diff options
| author | Dana Robinson <43805+derobins@users.noreply.github.com> | 2022-08-02 19:54:40 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-08-02 19:54:40 (GMT) |
| commit | fcf41b3cd60df51af9be529e379a9dd6c488d088 (patch) | |
| tree | e486d5f8254a33b978c34069b9810ce171ba7c2c | |
| parent | ea13de1bb0aba8a97c75f10343dc4c792193b215 (diff) | |
| download | hdf5-fcf41b3cd60df51af9be529e379a9dd6c488d088.zip hdf5-fcf41b3cd60df51af9be529e379a9dd6c488d088.tar.gz hdf5-fcf41b3cd60df51af9be529e379a9dd6c488d088.tar.bz2 | |
Onion VFD (#1953)
* Onion VFD feature
* Fixes onion VFD errors with non-sec2 backing store VFDs
* Disables the onion VFD tests w/ ph5diff
* Disables non-sec2 VFDs as onion VFD backing stores
* Committing clang-format changes
* Formatted source
* Typo
* Adds onion VFD tools tests to CMake
* Fixes for v16 API compatibility
* Memset structs to avoid bad frees on errors
* H5Dwrite() calls now use H5T_NATIVE_INT as the memory type vs LE
* Properly decodes checksums on BE machines
* Be more careful about uint64_t to haddr_t/hsize_t conversions
* Another fix for BE data comparison
* Removed double underscores from onion constants
* Replace hard-coded onion header string w/ constant
* Fixes cleanup paths in H5FD__onion_ingest_history()
* Fixed use of size_t revision numbers
* Fix h5dump revision count format string
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
55 files changed, 10632 insertions, 39 deletions
@@ -200,6 +200,7 @@ $Source = ""; "H5FD_t" => "#", "H5FD_hdfs_fapl_t" => "#", "H5FD_mirror_fapl_t" => "#", + "H5FD_onion_fapl_t" => "#", "H5FD_ros3_fapl_t" => "#", "H5FD_splitter_vfd_config_t" => "#", "H5L_class_t" => "#", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b98874..a835ebd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -240,6 +240,10 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDmpi.c ${HDF5_SRC_DIR}/H5FDmpio.c ${HDF5_SRC_DIR}/H5FDmulti.c + ${HDF5_SRC_DIR}/H5FDonion.c + ${HDF5_SRC_DIR}/H5FDonion_header.c + ${HDF5_SRC_DIR}/H5FDonion_history.c + ${HDF5_SRC_DIR}/H5FDonion_index.c ${HDF5_SRC_DIR}/H5FDperform.c ${HDF5_SRC_DIR}/H5FDros3.c ${HDF5_SRC_DIR}/H5FDs3comms.c @@ -262,6 +266,7 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDmpi.h ${HDF5_SRC_DIR}/H5FDmpio.h ${HDF5_SRC_DIR}/H5FDmulti.h + ${HDF5_SRC_DIR}/H5FDonion.h ${HDF5_SRC_DIR}/H5FDpublic.h ${HDF5_SRC_DIR}/H5FDros3.h ${HDF5_SRC_DIR}/H5FDs3comms.h @@ -902,6 +907,10 @@ set (H5_PRIVATE_HEADERS ${HDF5_SRC_DIR}/H5FAprivate.h ${HDF5_SRC_DIR}/H5FDmirror_priv.h + ${HDF5_SRC_DIR}/H5FDonion_header.h + ${HDF5_SRC_DIR}/H5FDonion_history.h + ${HDF5_SRC_DIR}/H5FDonion_index.h + ${HDF5_SRC_DIR}/H5FDonion_priv.h ${HDF5_SRC_DIR}/H5FDpkg.h ${HDF5_SRC_DIR}/H5FDprivate.h diff --git a/src/H5FDonion.c b/src/H5FDonion.c new file mode 100644 index 0000000..23ea624 --- /dev/null +++ b/src/H5FDonion.c @@ -0,0 +1,1762 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Provide in-file provenance and revision/version control. + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDonion.h" /* Onion file driver */ +#include "H5FDonion_priv.h" /* Onion file driver internals */ +#include "H5FDsec2.h" /* Sec2 file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_ONION_g = 0; + +/****************************************************************************** + * + * Structure: H5FD_onion_t + * + * Purpose: Store information required to manage an onionized file. + * This structure is created when such a file is "opened" and + * discarded when it is "closed". + * + * pu + * + * Instance of H5FD_t which contains fields common to all VFDs. + * It must be the first item in this structure, since at higher levels, + * this structure will be treated as an instance of H5FD_t. + * + * fa + * + * Instance of `H5FD_onion_fapl_info_t` containing the configuration data + * needed to "open" the HDF5 file. + * + * original_file + * + * VFD handle for the original HDF5 file. + * + * onion_file + * + * VFD handle for the onion file. + * NULL if not set to use the single, separate storage target. + * + * recovery_file + * + * VFD handle for the history recovery file. This file is a backup of + * the existing history when an existing onion file is opened in RW mode. + * + * recovery_file_name + * + * String allocated and populated on file-open in write mode and freed on + * file-close, stores the path/name of the 'recovery' file. The file + * created at this location is to be removed upon successful file-close + * from write mode. + * + * is_open_rw + * + * Remember whether the file was opened in a read-write mode. + * + * align_history_on_pages + * + * Remember whether onion-writes must be aligned to page boundaries. + * + * header + * + * In-memory copy of the onion history data header. + * + * history + * + * In-memory copy of the onion history. + * + * curr_rev_record + * + * Record for the currently open revision. + * + * rev_index + * + * Index for maintaining modified pages (RW mode only). + * Pointer is NULL when the file is not opened in write mode. + * Pointer is allocated on open and must be freed on close. + * Contents must be merged with the revision record's archival index prior + * to commitment of history to backing store. + * + * onion_eof + * + * Last byte in the onion file. + * + * origin_eof + * + * Size of the original HDF5 file. + * + * logical_eoa + * + * Address of first byte past addressed space in the logical 'file' + * presented by this VFD. + * + * logical_eof + * + * Address of first byte past last byte in the logical 'file' presented + * by this VFD. + * Must be copied into the revision record on close to write onion data. + * + ****************************************************************************** + */ +typedef struct H5FD_onion_t { + H5FD_t pub; + H5FD_onion_fapl_info_t fa; + hbool_t is_open_rw; + hbool_t align_history_on_pages; + + /* Onion-related files */ + H5FD_t *original_file; + H5FD_t *onion_file; + H5FD_t *recovery_file; + char *recovery_file_name; + + /* Onion data structures */ + H5FD_onion_header_t header; + H5FD_onion_history_t history; + H5FD_onion_revision_record_t curr_rev_record; + H5FD_onion_revision_index_t *rev_index; + + /* End of addresses and files */ + haddr_t onion_eof; + haddr_t origin_eof; + haddr_t logical_eoa; + haddr_t logical_eof; +} H5FD_onion_t; + +H5FL_DEFINE_STATIC(H5FD_onion_t); + +#define MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1) + +#define H5FD_CTL_GET_NUM_REVISIONS 20001 + +/* Prototypes */ +static herr_t H5FD__onion_close(H5FD_t *); +static haddr_t H5FD__onion_get_eoa(const H5FD_t *, H5FD_mem_t); +static haddr_t H5FD__onion_get_eof(const H5FD_t *, H5FD_mem_t); +static H5FD_t *H5FD__onion_open(const char *, unsigned int, hid_t, haddr_t); +static herr_t H5FD__onion_read(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, void *); +static herr_t H5FD__onion_set_eoa(H5FD_t *, H5FD_mem_t, haddr_t); +static herr_t H5FD__onion_term(void); +static herr_t H5FD__onion_write(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, const void *); + +static herr_t H5FD__onion_open_rw(H5FD_onion_t *, unsigned int, haddr_t, bool new_open); +static herr_t H5FD__onion_sb_encode(H5FD_t *_file, char *name /*out*/, unsigned char *buf /*out*/); +static herr_t H5FD__onion_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf); +static hsize_t H5FD__onion_sb_size(H5FD_t *_file); +static herr_t H5FD__onion_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, + const void H5_ATTR_UNUSED *input, void H5_ATTR_UNUSED **output); +static herr_t H5FD__get_onion_revision_count(H5FD_t *file, uint64_t *revision_count); + +/* Temporary */ +H5_DLL herr_t H5FD__onion_write_final_history(H5FD_onion_t *file); + +static const H5FD_class_t H5FD_onion_g = { + H5FD_CLASS_VERSION, /* struct version */ + H5FD_ONION_VALUE, /* value */ + "onion", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__onion_term, /* terminate */ + H5FD__onion_sb_size, /* sb_size */ + H5FD__onion_sb_encode, /* sb_encode */ + H5FD__onion_sb_decode, /* sb_decode */ + sizeof(H5FD_onion_fapl_info_t), /* fapl_size */ + NULL, /* fapl_get */ + NULL, /* fapl_copy */ + NULL, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__onion_open, /* open */ + H5FD__onion_close, /* close */ + NULL, /* cmp */ + NULL, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD__onion_get_eoa, /* get_eoa */ + H5FD__onion_set_eoa, /* set_eoa */ + H5FD__onion_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD__onion_read, /* read */ + H5FD__onion_write, /* write */ + NULL, /* read_vector */ + NULL, /* write_vector */ + NULL, /* read_selection */ + NULL, /* write_selection */ + NULL, /* flush */ + NULL, /* truncate */ + NULL, /* lock */ + NULL, /* unlock */ + NULL, /* del */ + H5FD__onion_ctl, /* ctl */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/*----------------------------------------------------------------------------- + * Function: H5FD_onion_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the onion driver. + * Failure: Negative + *----------------------------------------------------------------------------- + */ +hid_t +H5FD_onion_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI_NOERR + + if (H5I_VFL != H5I_get_type(H5FD_ONION_g)) + H5FD_ONION_g = H5FD_register(&H5FD_onion_g, sizeof(H5FD_class_t), FALSE); + + /* Set return value */ + ret_value = H5FD_ONION_g; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_onion_init() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_term + * + * Purpose: Shut down the Onion VFD. + * + * Returns: SUCCEED (Can't fail) + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_term(void) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* Reset VFL ID */ + H5FD_ONION_g = 0; + + FUNC_LEAVE_NOAPI(SUCCEED); + +} /* end H5FD__onion_term() */ + +/*----------------------------------------------------------------------------- + * Function: H5Pget_fapl_onion + * + * Purpose: Copy the Onion configuration information from the FAPL at + * `fapl_id` to the destination pointer `fa_out`. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + *----------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out) +{ + const H5FD_onion_fapl_info_t *info_ptr = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*!", fapl_id, fa_out); + + if (NULL == fa_out) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info-out pointer") + + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID") + + if (H5FD_ONION != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Incorrect VFL driver") + + if (NULL == (info_ptr = (const H5FD_onion_fapl_info_t *)H5P_peek_driver_info(plist))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad VFL driver info") + + HDmemcpy(fa_out, info_ptr, sizeof(H5FD_onion_fapl_info_t)); + +done: + FUNC_LEAVE_API(ret_value) + +} /* end H5Pget_fapl_onion() */ + +/*----------------------------------------------------------------------------- + * Function: H5Pset_fapl_onion + * + * Purpose Set the file access property list at `fapl_id` to use the + * Onion virtual file driver with the given configuration. + * The info structure may be modified or deleted after this call, + * as its contents are copied into the FAPL. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + *----------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa) +{ + H5P_genplist_t *fapl = NULL; + H5P_genplist_t *backing_fapl = NULL; + hid_t backing_vfd_id = H5I_INVALID_HID; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*!", fapl_id, fa); + + if (NULL == (fapl = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID"); + if (NULL == fa) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info pointer"); + if (H5FD_ONION_FAPL_INFO_VERSION_CURR != fa->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info version"); + if (!POWER_OF_TWO(fa->page_size)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size"); + if (fa->page_size < 1) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size"); + + if (H5P_DEFAULT == fa->backing_fapl_id) { + if (NULL == (backing_fapl = H5P_object_verify(H5P_FILE_ACCESS_DEFAULT, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id"); + } + else { + if (NULL == (backing_fapl = H5P_object_verify(fa->backing_fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid backing fapl id"); + } + + /* The only backing fapl that is currently supported is sec2 */ + if ((backing_vfd_id = H5P_peek_driver(backing_fapl)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "Can't get VFD from fapl"); + if (backing_vfd_id != H5FD_SEC2) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "Onion VFD only supports sec2 backing store"); + + if (H5P_set_driver(fapl, H5FD_ONION, (const void *)fa, NULL) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "Can't set the onion VFD"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_onion() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__onion_sb_size + * + * Purpose: Returns the size of the private information to be stored in + * the superblock. + * + * Return: Success: The super block driver data size + * Failure: never fails + *------------------------------------------------------------------------- + */ +static hsize_t +H5FD__onion_sb_size(H5FD_t *_file) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + hsize_t ret_value = 0; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity check */ + HDassert(file); + HDassert(file->original_file); + + if (file->original_file) + ret_value = H5FD_sb_size(file->original_file); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__onion_sb_size */ + +/*------------------------------------------------------------------------- + * Function: H5FD__onion_sb_encode + * + * Purpose: Encodes the superblock information for this driver + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_sb_encode(H5FD_t *_file, char *name /*out*/, unsigned char *buf /*out*/) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(file); + HDassert(file->original_file); + + if (file->original_file && H5FD_sb_encode(file->original_file, name, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTENCODE, FAIL, "unable to encode the superblock in R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__onion_sb_encode */ + +/*------------------------------------------------------------------------- + * Function: H5FD__onion_sb_decode + * + * Purpose: Decodes the superblock information for this driver + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(file); + HDassert(file->original_file); + + if (H5FD_sb_load(file->original_file, name, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "unable to decode the superblock in R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__onion_sb_decode */ + +/*----------------------------------------------------------------------------- + * Write in-memory revision record to appropriate backing file. + * Update information in other in-memory components. + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_commit_new_revision_record(H5FD_onion_t *file) +{ + uint32_t checksum = 0; /* required */ + size_t size = 0; + haddr_t phys_addr = 0; /* offset in history file to record start */ + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + H5FD_onion_revision_record_t *rec = &file->curr_rev_record; + H5FD_onion_history_t *history = &file->history; + H5FD_onion_record_loc_t *new_list = NULL; + + time_t rawtime; + struct tm *info; + + FUNC_ENTER_PACKAGE + + HDtime(&rawtime); + info = HDgmtime(&rawtime); + HDstrftime(rec->time_of_creation, sizeof(rec->time_of_creation), "%Y%m%dT%H%M%SZ", info); + + rec->logical_eof = file->logical_eof; + + if ((TRUE == file->is_open_rw) && (H5FD__onion_merge_revision_index_into_archival_index( + file->rev_index, &file->curr_rev_record.archival_index) < 0)) + HGOTO_ERROR(H5E_VFL, H5E_INTERNAL, FAIL, "unable to update index to write") + + if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_REVISION_RECORD + (size_t)rec->comment_size + + (H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * rec->archival_index.n_entries)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for encoded revision record") + + if (0 == (size = H5FD__onion_revision_record_encode(rec, buf, &checksum))) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding revision record") + + phys_addr = file->onion_eof; + if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, phys_addr + size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA for new revision record") + if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, phys_addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write new revision record") + + file->onion_eof = phys_addr + size; + if (TRUE == file->align_history_on_pages) + file->onion_eof = (file->onion_eof + (file->header.page_size - 1)) & (~(file->header.page_size - 1)); + + /* Update history info to accommodate new revision */ + + if (history->n_revisions == 0) { + unsigned char *ptr = buf; /* re-use buffer space to compute checksum */ + + HDassert(history->record_locs == NULL); + history->n_revisions = 1; + if (NULL == (history->record_locs = H5MM_calloc(sizeof(H5FD_onion_record_loc_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate temporary record pointer list") + + history->record_locs[0].phys_addr = phys_addr; + history->record_locs[0].record_size = size; + UINT64ENCODE(ptr, phys_addr); + UINT64ENCODE(ptr, size); + history->record_locs[0].checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + /* TODO: size-reset belongs where? */ + file->header.history_size += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER; + } /* end if no extant revisions in history */ + else { + unsigned char *ptr = buf; /* re-use buffer space to compute checksum */ + + HDassert(history->record_locs != NULL); + + if (NULL == (new_list = H5MM_calloc((history->n_revisions + 1) * sizeof(H5FD_onion_record_loc_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to resize record pointer list") + HDmemcpy(new_list, history->record_locs, sizeof(H5FD_onion_record_loc_t) * history->n_revisions); + H5MM_xfree(history->record_locs); + history->record_locs = new_list; + new_list = NULL; + history->record_locs[history->n_revisions].phys_addr = phys_addr; + history->record_locs[history->n_revisions].record_size = size; + UINT64ENCODE(ptr, phys_addr); + UINT64ENCODE(ptr, size); + history->record_locs[history->n_revisions].checksum = + H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + file->header.history_size += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER; + history->n_revisions += 1; + } /* end if one or more revisions present in history */ + + file->header.history_addr = file->onion_eof; + +done: + H5MM_xfree(buf); + H5MM_xfree(new_list); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_commit_new_revision_record() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_close + * + * Purpose: Close an onionized file + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_close(H5FD_t *_file) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert(file); + + if (H5FD_ONION_STORE_TARGET_ONION == file->fa.store_target) { + + HDassert(file->onion_file); + + if (file->is_open_rw) { + + HDassert(file->recovery_file); + + if (H5FD__onion_commit_new_revision_record(file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write revision record to backing store") + + if (H5FD__onion_write_final_history(file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write history to backing store") + + /* Unset write-lock flag and write header */ + if (file->is_open_rw) + file->header.flags &= (uint32_t)~H5FD_ONION_HEADER_FLAG_WRITE_LOCK; + if (H5FD__onion_write_header(&(file->header), file->onion_file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "Can't write updated header to backing store") + } + } + else if (H5FD_ONION_STORE_TARGET_H5 == file->fa.store_target) + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "hdf5 store-target not supported") + else + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid history target") + +done: + + /* Destroy things as best we can, even if there were earlier errors */ + if (file->original_file) + if (H5FD_close(file->original_file) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing canon file") + if (file->onion_file) + if (H5FD_close(file->onion_file) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing onion file") + if (file->recovery_file) { + if (H5FD_close(file->recovery_file) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close backing recovery file") + /* TODO: Use the VFD's del callback instead of remove (this requires + * storing a copy of the fapl that was used to open it) + */ + HDremove(file->recovery_file_name); + } + if (file->rev_index) + if (H5FD__onion_revision_index_destroy(file->rev_index) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't close revision index") + + H5MM_xfree(file->recovery_file_name); + H5MM_xfree(file->history.record_locs); + H5MM_xfree(file->curr_rev_record.comment); + H5MM_xfree(file->curr_rev_record.archival_index.list); + + file = H5FL_FREE(H5FD_onion_t, file); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__onion_close() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_get_eoa + * + * Purpose: Get end-of-address address. + * + * Return: Address of first byte past the addressed space + *----------------------------------------------------------------------------- + */ +static haddr_t +H5FD__onion_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_onion_t *file = (const H5FD_onion_t *)_file; + + FUNC_ENTER_PACKAGE_NOERR; + + FUNC_LEAVE_NOAPI(file->logical_eoa) +} /* end H5FD__onion_get_eoa() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_get_eof + * + * Purpose: Get end-of-file address. + * + * Return: Address of first byte past the file-end. + *----------------------------------------------------------------------------- + */ +static haddr_t +H5FD__onion_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_onion_t *file = (const H5FD_onion_t *)_file; + + FUNC_ENTER_PACKAGE_NOERR; + + FUNC_LEAVE_NOAPI(file->logical_eof) +} /* end H5FD__onion_get_eof() */ + +/*----------------------------------------------------------------------------- + * Sanitize the backing FAPL ID + *----------------------------------------------------------------------------- + */ +static inline hid_t +H5FD__onion_get_legit_fapl_id(hid_t fapl_id) +{ + if (H5P_DEFAULT == fapl_id) + return H5P_FILE_ACCESS_DEFAULT; + else if (TRUE == H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + return fapl_id; + else + return H5I_INVALID_HID; +} + +/*----------------------------------------------------------------------------- + * Function: H5FD_onion_create_truncate_onion + * + * Purpose: Create/truncate HDF5 and onion data for a fresh file + * + * Special open operation required to instantiate the canonical file and + * history simultaneously. If successful, the required backing files are + * craeated and given initial population on the backing store, and the Onion + * virtual file handle is set; open effects a write-mode open. + * + * Cannot create 'template' history and proceed with normal write-mode open, + * as this would in effect create an empty first revision, making the history + * unintuitive. (create file -> initialize and commit empty first revision + * (revision 0); any data written to file during the 'create' open, as seen by + * the user, would be in the second revision (revision 1).) + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_create_truncate_onion(H5FD_onion_t *file, const char *filename, const char *name_onion, + const char *recovery_file_nameery, unsigned int flags, haddr_t maxaddr) +{ + hid_t backing_fapl_id = H5I_INVALID_HID; + H5FD_onion_header_t *hdr = NULL; + H5FD_onion_history_t *history = NULL; + H5FD_onion_revision_record_t *rec = NULL; + unsigned char *buf = NULL; + size_t size = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + HDassert(file != NULL); + + hdr = &file->header; + history = &file->history; + rec = &file->curr_rev_record; + + hdr->flags = H5FD_ONION_HEADER_FLAG_WRITE_LOCK; + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY & file->fa.creation_flags) + hdr->flags |= H5FD_ONION_HEADER_FLAG_DIVERGENT_HISTORY; + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & file->fa.creation_flags) + hdr->flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT; + + hdr->origin_eof = 0; + + backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id); + if (H5I_INVALID_HID == backing_fapl_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid backing FAPL ID") + + /* Create backing files for onion history */ + + if (NULL == (file->original_file = H5FD_open(filename, flags, backing_fapl_id, maxaddr))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "cannot open the backing file") + + if (NULL == (file->onion_file = H5FD_open(name_onion, flags, backing_fapl_id, maxaddr))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "cannot open the backing onion file") + + if (NULL == (file->recovery_file = H5FD_open(recovery_file_nameery, flags, backing_fapl_id, maxaddr))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "cannot open the backing file") + + /* Write "empty" .h5 file contents (signature ONIONEOF) */ + + if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, 8) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA") + if (H5FD_write(file->original_file, H5FD_MEM_DRAW, 0, 8, "ONIONEOF") < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "cannot write header to the backing h5 file") + + /* Write nascent history (with no revisions) to "recovery" */ + + if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer") + size = H5FD__onion_history_encode(history, buf, &history->checksum); + if (H5FD_ONION_ENCODED_SIZE_HISTORY != size) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history") + if (H5FD_set_eoa(file->recovery_file, H5FD_MEM_DRAW, size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA") + if (H5FD_write(file->recovery_file, H5FD_MEM_DRAW, 0, size, buf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "cannot write history to the backing recovery file") + hdr->history_size = size; /* record for later use */ + H5MM_xfree(buf); + buf = NULL; + + /* Write history header with "no" history. + * Size of the "recovery" history recorded for later use on close. + */ + + if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer") + size = H5FD__onion_header_encode(hdr, buf, &hdr->checksum); + if (H5FD_ONION_ENCODED_SIZE_HEADER != size) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history header") + if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA") + if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, size, buf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "cannot write header to the backing onion file") + file->onion_eof = (haddr_t)size; + if (TRUE == file->align_history_on_pages) + file->onion_eof = (file->onion_eof + (hdr->page_size - 1)) & (~(hdr->page_size - 1)); + + rec->archival_index.list = NULL; + + if (NULL == (file->rev_index = H5FD__onion_revision_index_init(file->fa.page_size))) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize revision index") + +done: + H5MM_xfree(buf); + + if (FAIL == ret_value) + HDremove(recovery_file_nameery); /* destroy new temp file, if 'twas created */ + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_create_truncate_onion() */ + +static herr_t +H5FD__onion_remove_unused_symbols(char *s) +{ + char *d = s; + + FUNC_ENTER_PACKAGE_NOERR; + + do { + while (*d == '{' || *d == '}' || *d == ' ') { + ++d; + } + } while ((*s++ = *d++)); + + FUNC_LEAVE_NOAPI(SUCCEED); +} + +static herr_t +H5FD__onion_parse_config_str(const char *config_str, H5FD_onion_fapl_info_t *fa) +{ + char *config_str_copy = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + if (!HDstrcmp(config_str, "")) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "configure string can't be empty") + + /* Initialize to the default values */ + fa->version = H5FD_ONION_FAPL_INFO_VERSION_CURR; + fa->backing_fapl_id = H5P_DEFAULT; + fa->page_size = 4; + fa->store_target = H5FD_ONION_STORE_TARGET_ONION; + fa->revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + fa->force_write_open = 0; + fa->creation_flags = 0; + HDstrcpy(fa->comment, "initial comment"); + + /* If a single integer is passed in as a string, it's a shortcut for the tools + * (h5repack, h5diff, h5dump). Otherwise, the string should have curly brackets, + * e.g. {revision_num: 2; page_size: 4;} + */ + if (config_str[0] != '{') + fa->revision_num = (uint64_t)HDstrtoull(config_str, NULL, 10); + else { + char *token1 = NULL, *token2 = NULL; + + /* Duplicate the configure string since strtok will mess with it */ + if (NULL == (config_str_copy = H5MM_strdup(config_str))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't duplicate configure string") + + /* Remove the curly brackets and space from the configure string */ + H5FD__onion_remove_unused_symbols(config_str_copy); + + /* The configure string can't be empty after removing the curly brackets */ + if (!HDstrcmp(config_str_copy, "")) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "configure string can't be empty") + + token1 = HDstrtok(config_str_copy, ":"); + token2 = HDstrtok(NULL, ";"); + + do { + if (token1 && token2) { + if (!HDstrcmp(token1, "version")) { + if (!HDstrcmp(token2, "H5FD_ONION_FAPL_INFO_VERSION_CURR")) + fa->version = H5FD_ONION_FAPL_INFO_VERSION_CURR; + } + else if (!HDstrcmp(token1, "backing_fapl_id")) { + if (!HDstrcmp(token2, "H5P_DEFAULT")) + fa->backing_fapl_id = H5P_DEFAULT; + else if (!strcmp(token2, "H5I_INVALID_HID")) + fa->backing_fapl_id = H5I_INVALID_HID; + else + fa->backing_fapl_id = HDstrtoll(token2, NULL, 10); + } + else if (!HDstrcmp(token1, "page_size")) { + fa->page_size = (uint32_t)HDstrtoul(token2, NULL, 10); + } + else if (!HDstrcmp(token1, "revision_num")) { + if (!HDstrcmp(token2, "H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST")) + fa->revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + else + fa->revision_num = (uint64_t)HDstrtoull(token2, NULL, 10); + } + else if (!HDstrcmp(token1, "force_write_open")) { + fa->force_write_open = (uint8_t)HDstrtoul(token2, NULL, 10); + } + else if (!HDstrcmp(token1, "creation_flags")) { + fa->creation_flags = (uint8_t)HDstrtoul(token2, NULL, 10); + } + else if (!HDstrcmp(token1, "comment")) { + HDstrcpy(fa->comment, token2); + } + else + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unknown token in the configure string: %s", + token1) + } + + token1 = HDstrtok(NULL, ":"); + token2 = HDstrtok(NULL, ";"); + } while (token1); + } + + if (H5P_DEFAULT == fa->backing_fapl_id || H5I_INVALID_HID == fa->backing_fapl_id) { + H5P_genclass_t *pclass; /* Property list class to modify */ + + if (NULL == (pclass = (H5P_genclass_t *)H5I_object_verify(H5P_FILE_ACCESS, H5I_GENPROP_CLS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a property list class"); + + /* Create the new property list */ + if ((fa->backing_fapl_id = H5P_create_id(pclass, TRUE)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, FAIL, "unable to create property list"); + } + +done: + H5MM_free(config_str_copy); + + FUNC_LEAVE_NOAPI(ret_value); +} + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_open + * + * Purpose: Open an onionized file + * + * Return: Success: A pointer to a new file data structure + * Failure: NULL + *----------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD__onion_open(const char *filename, unsigned flags, hid_t fapl_id, haddr_t maxaddr) +{ + H5P_genplist_t *plist = NULL; + H5FD_onion_t *file = NULL; + const H5FD_onion_fapl_info_t *fa = NULL; + H5FD_onion_fapl_info_t *new_fa = NULL; + const char *config_str = NULL; + hid_t backing_fapl_id = H5I_INVALID_HID; + char *name_onion = NULL; + char *recovery_file_nameery = NULL; + bool new_open = false; + haddr_t canon_eof = 0; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_PACKAGE + + /* Check arguments */ + if (!filename || !*filename) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + if (0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + HDassert(H5P_DEFAULT != fapl_id); + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") + + /* This VFD can be invoked by either H5Pset_fapl_onion() or + * H5Pset_driver_by_name(). When invoked by the former, there will be + * driver info to peek at. + */ + fa = (const H5FD_onion_fapl_info_t *)H5P_peek_driver_info(plist); + + if (NULL == fa) { + if (NULL == (config_str = H5P_peek_driver_config_str(plist))) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "missing VFL driver configure string") + + /* Allocate a new onion fapl info struct and fill it from the + * configuration string + */ + if (NULL == (new_fa = H5MM_calloc(sizeof(H5FD_onion_fapl_info_t)))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate memory for onion fapl info struct") + if (H5FD__onion_parse_config_str(config_str, new_fa) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "failed to parse configure string") + + fa = new_fa; + } + + /* Check for unsupported target values */ + if (H5FD_ONION_STORE_TARGET_H5 == fa->store_target) + HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "same-file storage not implemented") + else if (H5FD_ONION_STORE_TARGET_ONION != fa->store_target) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid store target") + + /* Allocate space for the file struct */ + if (NULL == (file = H5FL_CALLOC(H5FD_onion_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate file struct") + + /* Allocate space for onion VFD file names */ + if (NULL == (name_onion = H5MM_malloc(sizeof(char) * (HDstrlen(filename) + 7)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate onion name string") + HDsnprintf(name_onion, HDstrlen(filename) + 7, "%s.onion", filename); + + if (NULL == (recovery_file_nameery = H5MM_malloc(sizeof(char) * (HDstrlen(name_onion) + 10)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate recovery name string") + HDsnprintf(recovery_file_nameery, HDstrlen(name_onion) + 10, "%s.recovery", name_onion); + file->recovery_file_name = recovery_file_nameery; + + if (NULL == (file->recovery_file_name = H5MM_malloc(sizeof(char) * (HDstrlen(name_onion) + 10)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate recovery name string") + HDsnprintf(file->recovery_file_name, HDstrlen(name_onion) + 10, "%s.recovery", name_onion); + + /* Translate H5P_DEFAULT to a a real fapl ID, if necessary */ + backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id); + if (H5I_INVALID_HID == backing_fapl_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid backing FAPL ID"); + + /* Initialize file structure fields */ + + HDmemcpy(&(file->fa), fa, sizeof(H5FD_onion_fapl_info_t)); + + file->header.version = H5FD_ONION_HEADER_VERSION_CURR; + file->header.page_size = file->fa.page_size; /* guarded on FAPL-set */ + + file->history.version = H5FD_ONION_HISTORY_VERSION_CURR; + + file->curr_rev_record.version = H5FD_ONION_REVISION_RECORD_VERSION_CURR; + file->curr_rev_record.archival_index.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR; + + /* Check that the page size is a power of two */ + if ((fa->page_size == 0) || ((fa->page_size & (fa->page_size - 1)) != 0)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "page size is not a power of two") + + /* Assign the page size */ + double log2_page_size = HDlog2((double)(fa->page_size)); + file->curr_rev_record.archival_index.page_size_log2 = (uint32_t)log2_page_size; + + /* Proceed with open. */ + + if ((H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) { + + /* Create a new onion file from scratch */ + + /* Set flags */ + if (fa->creation_flags & H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT) { + file->header.flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT; + file->align_history_on_pages = TRUE; + } + + /* Truncate and create everything as necessary */ + if (H5FD__onion_create_truncate_onion(file, filename, name_onion, file->recovery_file_name, flags, + maxaddr) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, "unable to create/truncate onionized files") + file->is_open_rw = TRUE; + } + else { + + /* Opening an existing onion file */ + + /* Open the existing file using the specified fapl */ + if (NULL == (file->original_file = H5FD_open(filename, flags, backing_fapl_id, maxaddr))) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open canonical file (does not exist?)") + + /* Try to open any existing onion file */ + H5E_BEGIN_TRY + { + file->onion_file = H5FD_open(name_onion, flags, backing_fapl_id, maxaddr); + } + H5E_END_TRY; + + /* If that didn't work, create a new onion file */ + /* TODO: Move to a new function */ + if (NULL == file->onion_file) { + if (H5F_ACC_RDWR & flags) { + H5FD_onion_header_t *hdr = NULL; + H5FD_onion_history_t *history = NULL; + H5FD_onion_revision_record_t *rec = NULL; + unsigned char *head_buf = NULL; + unsigned char *hist_buf = NULL; + size_t size = 0; + size_t saved_size = 0; + + HDassert(file != NULL); + + hdr = &file->header; + history = &file->history; + rec = &file->curr_rev_record; + + new_open = true; + + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY & file->fa.creation_flags) + hdr->flags |= H5FD_ONION_HEADER_FLAG_DIVERGENT_HISTORY; + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & file->fa.creation_flags) { + hdr->flags |= H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT; + file->align_history_on_pages = TRUE; + } + + if (HADDR_UNDEF == (canon_eof = H5FD_get_eof(file->original_file, H5FD_MEM_DEFAULT))) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "cannot get size of canonical file") + } + if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, canon_eof) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't extend EOA") + hdr->origin_eof = canon_eof; + file->logical_eof = canon_eof; + + backing_fapl_id = H5FD__onion_get_legit_fapl_id(file->fa.backing_fapl_id); + + if (H5I_INVALID_HID == backing_fapl_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid backing FAPL ID") + + /* Create backing files for onion history */ + + if ((file->onion_file = H5FD_open(name_onion, (H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC), + backing_fapl_id, maxaddr)) == NULL) { + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "cannot open the backing onion file") + } + + /* Write history header with "no" history */ + hdr->history_size = H5FD_ONION_ENCODED_SIZE_HISTORY; /* record for later use */ + hdr->history_addr = + H5FD_ONION_ENCODED_SIZE_HEADER + 1; /* TODO: comment these 2 or do some other way */ + head_buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER); + if (NULL == head_buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't allocate buffer") + size = H5FD__onion_header_encode(hdr, head_buf, &hdr->checksum); + if (H5FD_ONION_ENCODED_SIZE_HEADER != size) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't encode history header") + + hist_buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY); + if (NULL == hist_buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't allocate buffer") + saved_size = size; + history->n_revisions = 0; + size = H5FD__onion_history_encode(history, hist_buf, &history->checksum); + file->header.history_size = size; /* record for later use */ + if (H5FD_ONION_ENCODED_SIZE_HISTORY != size) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't encode history") + } + if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, saved_size + size + 1) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't extend EOA") + + if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, saved_size, head_buf) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, NULL, + "cannot write header to the backing onion file") + } + + file->onion_eof = (haddr_t)saved_size; + if (TRUE == file->align_history_on_pages) + file->onion_eof = (file->onion_eof + (hdr->page_size - 1)) & (~(hdr->page_size - 1)); + + rec->archival_index.list = NULL; + + file->header.history_addr = file->onion_eof; + + /* Write nascent history (with no revisions) to the backing onion file */ + if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, saved_size + 1, size, hist_buf) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, NULL, + "cannot write history to the backing onion file") + } + + file->header.history_size = size; /* record for later use */ + + H5MM_xfree(head_buf); + H5MM_xfree(hist_buf); + } + else { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open onion file (does not exist?).") + } + } + + if (HADDR_UNDEF == (canon_eof = H5FD_get_eof(file->original_file, H5FD_MEM_DEFAULT))) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "cannot get size of canonical file") + } + if (H5FD_set_eoa(file->original_file, H5FD_MEM_DRAW, canon_eof) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't extend EOA") + + /* Get the history header from the onion file */ + if (H5FD__onion_ingest_header(&file->header, file->onion_file, 0) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get history header from backing store") + file->align_history_on_pages = + (file->header.flags & H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT) ? TRUE : FALSE; + + if (H5FD_ONION_HEADER_FLAG_WRITE_LOCK & file->header.flags) { + /* Opening a file twice in write mode is an error */ + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "Can't open file already opened in write-mode") + } + else { + /* Read in the history from the onion file */ + if (H5FD__onion_ingest_history(&file->history, file->onion_file, file->header.history_addr, + file->header.history_size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get history from backing store") + + /* Sanity check on revision ID */ + if (fa->revision_num > file->history.n_revisions && + fa->revision_num != H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "target revision ID out of range") + + if (fa->revision_num == 0) { + file->curr_rev_record.logical_eof = canon_eof; + } + else if (file->history.n_revisions > 0 && + H5FD__onion_ingest_revision_record( + &file->curr_rev_record, file->onion_file, &file->history, + MIN(fa->revision_num - 1, (file->history.n_revisions - 1))) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, "can't get revision record from backing store") + } + + if (H5F_ACC_RDWR & flags) + if (H5FD__onion_open_rw(file, flags, maxaddr, new_open) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "can't write-open write-locked file") + } + + } /* End if opening existing file */ + + /* Copy comment from FAPL info, if one is given */ + if ((H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) { + if (fa->comment) { + /* Free the old comment */ + file->curr_rev_record.comment = H5MM_xfree(file->curr_rev_record.comment); + + /* TODO: Lengths of strings should be size_t */ + file->curr_rev_record.comment_size = (uint32_t)HDstrlen(fa->comment) + 1; + + if (NULL == (file->curr_rev_record.comment = H5MM_xstrdup(fa->comment))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "unable to allocate comment string") + } + } + file->origin_eof = file->header.origin_eof; + file->logical_eof = MAX(file->curr_rev_record.logical_eof, file->logical_eof); + file->logical_eoa = 0; + + file->onion_eof = H5FD_get_eoa(file->onion_file, H5FD_MEM_DRAW); + if (TRUE == file->align_history_on_pages) + file->onion_eof = (file->onion_eof + (file->header.page_size - 1)) & (~(file->header.page_size - 1)); + + ret_value = (H5FD_t *)file; + +done: + H5MM_xfree(name_onion); + H5MM_xfree(recovery_file_nameery); + + if (config_str && new_fa) + if (fa && fa->backing_fapl_id) + if (H5I_GENPROP_LST == H5I_get_type(fa->backing_fapl_id)) + H5I_dec_app_ref(fa->backing_fapl_id); + + if ((NULL == ret_value) && file) { + + if (file->original_file) + if (H5FD_close(file->original_file) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing canon") + if (file->onion_file) + if (H5FD_close(file->onion_file) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing onion") + if (file->recovery_file) + if (H5FD_close(file->recovery_file) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy backing recov") + + if (file->rev_index) + if (H5FD__onion_revision_index_destroy(file->rev_index) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, "can't destroy revision index") + + H5MM_xfree(file->history.record_locs); + + H5MM_xfree(file->recovery_file_name); + H5MM_xfree(file->curr_rev_record.comment); + + H5FL_FREE(H5FD_onion_t, file); + } + + H5MM_xfree(new_fa); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_open() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_open_rw + * + * Purpose: Complete onion file-open, handling process for write mode. + * + * Creates recovery file if one does not exist. + * Initializes 'live' revision index. + * Force write-open is not yet supported (recovery provision) TODO + * Establishes write-lock in history header (sets lock flag). + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_open_rw(H5FD_onion_t *file, unsigned int flags, haddr_t maxaddr, bool new_open) +{ + unsigned char *buf = NULL; + size_t size = 0; + uint32_t checksum = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + /* Guard against simultaneous write-open. + * TODO: support recovery open with force-write-open flag in FAPL info. + */ + + if (file->header.flags & H5FD_ONION_HEADER_FLAG_WRITE_LOCK) + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "can't write-open write-locked file") + + /* Copy history to recovery file */ + + if (NULL == + (file->recovery_file = H5FD_open(file->recovery_file_name, (flags | H5F_ACC_CREAT | H5F_ACC_TRUNC), + file->fa.backing_fapl_id, maxaddr))) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "unable to create recovery file") + + if (0 == (size = H5FD__onion_write_history(&file->history, file->recovery_file, 0, 0))) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write history to recovery file") + if (size != file->header.history_size) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "written history differed from expected size") + + /* Set write-lock flag in onion header */ + + if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate space for encoded buffer") + + file->header.flags |= H5FD_ONION_HEADER_FLAG_WRITE_LOCK; + + if (0 == (size = H5FD__onion_header_encode(&file->header, buf, &checksum))) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding history header") + + if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, 0, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write updated history header") + + /* Prepare revision index and finalize write-mode open */ + + if (NULL == (file->rev_index = H5FD__onion_revision_index_init(file->fa.page_size))) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize revision index") + file->curr_rev_record.parent_revision_num = file->curr_rev_record.revision_num; + if (!new_open) + file->curr_rev_record.revision_num += 1; + file->is_open_rw = TRUE; + +done: + if (FAIL == ret_value) { + if (file->recovery_file != NULL) { + if (H5FD_close(file->recovery_file) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close recovery file") + file->recovery_file = NULL; + } + + if (file->rev_index != NULL) { + if (H5FD__onion_revision_index_destroy(file->rev_index) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, "can't destroy revision index") + file->rev_index = NULL; + } + } + + H5MM_xfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_open_rw() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_read + * + * Purpose: Read bytes from an onionized file + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len, + void *_buf_out) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + uint64_t page_0 = 0; + size_t n_pages = 0; + uint32_t page_size = 0; + uint32_t page_size_log2 = 0; + size_t bytes_to_read = len; + unsigned char *buf_out = (unsigned char *)_buf_out; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert(file != NULL); + HDassert(buf_out != NULL); + + if ((uint64_t)(offset + len) > file->logical_eoa) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Read extends beyond addressed space") + + if (0 == len) + goto done; + + page_size = file->header.page_size; + page_size_log2 = file->curr_rev_record.archival_index.page_size_log2; + page_0 = offset >> page_size_log2; + n_pages = (len + page_size - 1) >> page_size_log2; + + /* Read, page-by-page */ + for (size_t i = 0; i < n_pages; i++) { + const H5FD_onion_index_entry_t *entry_out = NULL; + haddr_t page_gap_head = 0; /* start of page to start of buffer */ + haddr_t page_gap_tail = 0; /* end of buffer to end of page */ + size_t page_readsize = 0; + uint64_t page_i = page_0 + i; + + if (0 == i) { + page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1); + /* Check if we need to add an additional page to make up for the page_gap_head */ + if (page_gap_head > 0 && + (page_gap_head + (bytes_to_read % page_size) > page_size || bytes_to_read % page_size == 0)) { + n_pages++; + } + } + + if (n_pages - 1 == i) + page_gap_tail = page_size - bytes_to_read - page_gap_head; + + page_readsize = (size_t)page_size - page_gap_head - page_gap_tail; + + if (TRUE == file->is_open_rw && file->fa.revision_num != 0 && + H5FD__onion_revision_index_find(file->rev_index, page_i, &entry_out)) { + /* Page exists in 'live' revision index */ + if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr + page_gap_head, + page_readsize, buf_out) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get working file data") + } + else if (file->fa.revision_num != 0 && + H5FD__onion_archival_index_find(&file->curr_rev_record.archival_index, page_i, &entry_out)) { + /* Page exists in archival index */ + if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr + page_gap_head, + page_readsize, buf_out) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get previously-amended file data") + } + else { + /* Page does not exist in either index */ + + /* Casts prevent truncation */ + haddr_t addr_start = (haddr_t)page_i * (haddr_t)page_size + (haddr_t)page_gap_head; + haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 : file->origin_eof - addr_start; + haddr_t read_size = MIN(overlap_size, page_readsize); + + /* Get all original bytes in page range */ + if ((read_size > 0) && H5FD_read(file->original_file, type, addr_start, read_size, buf_out) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get original file data") + } + + /* Fill with 0s any gaps after end of original bytes + * and before end of page. + */ + for (size_t j = read_size; j < page_readsize; j++) + buf_out[j] = 0; + } + + buf_out += page_readsize; + bytes_to_read -= page_readsize; + } /* end for each page in range */ + + HDassert(0 == bytes_to_read); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_read() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_set_eoa + * + * Purpose: Set end-of-address marker of the logical file. + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + + FUNC_ENTER_PACKAGE_NOERR; + + file->logical_eoa = addr; + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* end H5FD__onion_set_eoa() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_write + * + * Purpose: Write bytes to an onionized file + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len, + const void *_buf) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + uint64_t page_0 = 0; + size_t n_pages = 0; + unsigned char *page_buf = NULL; + uint32_t page_size = 0; + uint32_t page_size_log2 = 0; + size_t bytes_to_write = len; + const unsigned char *buf = (const unsigned char *)_buf; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert(file != NULL); + HDassert(buf != NULL); + HDassert(file->rev_index != NULL); + HDassert((uint64_t)(offset + len) <= file->logical_eoa); + + if (FALSE == file->is_open_rw) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Write not allowed if file not opened in write mode") + + if (0 == len) + goto done; + + page_size = file->header.page_size; + page_size_log2 = file->curr_rev_record.archival_index.page_size_log2; + page_0 = offset >> page_size_log2; + n_pages = (len + page_size - 1) >> page_size_log2; + + if (NULL == (page_buf = H5MM_calloc(page_size * sizeof(unsigned char)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate temporary buffer") + + /* Write, page-by-page */ + for (size_t i = 0; i < n_pages; i++) { + const unsigned char *write_buf = buf; + H5FD_onion_index_entry_t new_entry; + const H5FD_onion_index_entry_t *entry_out = NULL; + haddr_t page_gap_head = 0; /* start of page to start of buffer */ + haddr_t page_gap_tail = 0; /* end of buffer to end of page */ + size_t page_n_used = 0; /* nbytes from buffer for this page-write */ + uint64_t page_i = page_0 + i; + + if (0 == i) { + page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1); + /* If we have a page_gap_head and the number of bytes to write is + * evenly divisible by the page size we need to add an additional + * page to make up for the page_gap_head + */ + if (page_gap_head > 0 && (page_gap_head + (bytes_to_write % page_size) > page_size || + bytes_to_write % page_size == 0)) { + n_pages++; + } + } + if (n_pages - 1 == i) + page_gap_tail = page_size - bytes_to_write - page_gap_head; + page_n_used = page_size - page_gap_head - page_gap_tail; + + /* Modify page in revision index, if present */ + if (H5FD__onion_revision_index_find(file->rev_index, page_i, &entry_out)) { + if (page_gap_head | page_gap_tail) { + /* Copy existing page verbatim. */ + if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, page_buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get working file data") + /* Overlay delta from input buffer onto page buffer. */ + HDmemcpy(page_buf + page_gap_head, buf, page_n_used); + write_buf = page_buf; + } /* end if partial page */ + + if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, write_buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write amended page data to backing file") + + buf += page_n_used; /* overflow never touched */ + bytes_to_write -= page_n_used; + + continue; + } /* end if page exists in 'live' revision index */ + + if (page_gap_head || page_gap_tail) { + /* Fill gaps with existing data or zeroes. */ + if (H5FD__onion_archival_index_find(&file->curr_rev_record.archival_index, page_i, &entry_out)) { + /* Page exists in archival index */ + + /* Copy existing page verbatim */ + if (H5FD_read(file->onion_file, H5FD_MEM_DRAW, entry_out->phys_addr, page_size, page_buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get previously-amended data") + } + else { + haddr_t addr_start = (haddr_t)(page_i * page_size); + haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 : file->origin_eof - addr_start; + haddr_t read_size = MIN(overlap_size, page_size); + + /* Get all original bytes in page range */ + if ((read_size > 0) && + H5FD_read(file->original_file, type, addr_start, read_size, page_buf) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't get original file data") + } + + /* Fill with 0s any gaps after end of original bytes + * or start of page and before start of new data. + */ + for (size_t j = read_size; j < page_gap_head; j++) + page_buf[j] = 0; + + /* Fill with 0s any gaps after end of original bytes + * or end of new data and before end of page. + */ + for (size_t j = MAX(read_size, page_size - page_gap_tail); j < page_size; j++) + page_buf[j] = 0; + } /* end if page exists in neither index */ + + /* Copy input buffer to temporary page buffer */ + HDassert((page_size - page_gap_head) >= page_n_used); + HDmemcpy(page_buf + page_gap_head, buf, page_n_used); + write_buf = page_buf; + + } /* end if data range does not span entire page */ + + new_entry.logical_page = page_i; + new_entry.phys_addr = file->onion_eof; + + if (H5FD_set_eoa(file->onion_file, H5FD_MEM_DRAW, file->onion_eof + page_size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA for new page amendment") + + if (H5FD_write(file->onion_file, H5FD_MEM_DRAW, file->onion_eof, page_size, write_buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write amended page data to backing file") + + if (H5FD__onion_revision_index_insert(file->rev_index, &new_entry) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINSERT, FAIL, "can't insert new index entry into revision index") + + file->onion_eof += page_size; + buf += page_n_used; /* possible overflow never touched */ + bytes_to_write -= page_n_used; + + } /* end for each page to write */ + + HDassert(0 == bytes_to_write); + + file->logical_eof = MAX(file->logical_eof, (offset + len)); + +done: + H5MM_xfree(page_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_write() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__onion_ctl + * + * Purpose: Onion VFD version of the ctl callback. + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input, + void **output) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(file); + + switch (op_code) { + case H5FD_CTL_GET_NUM_REVISIONS: + if (!output || !*output) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "the output parameter is null") + + **((uint64_t **)output) = file->history.n_revisions; + break; + /* Unknown op code */ + default: + if (flags & H5FD_CTL_FAIL_IF_UNKNOWN_FLAG) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set") + break; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__onion_ctl() */ + +/*------------------------------------------------------------------------- + * Function: H5FDget_onion_revision_count + * + * Purpose: Get the number of revisions in an onion file + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5FDonion_get_revision_count(const char *filename, hid_t fapl_id, uint64_t *revision_count /*out*/) +{ + H5P_genplist_t *plist = NULL; + H5FD_t *file = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "*six", filename, fapl_id, revision_count); + + /* Check args */ + if (!filename || !HDstrcmp(filename, "")) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid file name") + if (!revision_count) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "revision count can't be null") + + /* Make sure using the correct driver */ + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid FAPL ID") + if (H5FD_ONION != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a Onion VFL driver") + + /* Open the file with the driver */ + if (NULL == (file = H5FD_open(filename, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF))) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "unable to open file with onion driver") + + /* Call the private function */ + if (H5FD__get_onion_revision_count(file, revision_count) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "failed to get the number of revisions") + +done: + /* Close H5FD_t structure pointer */ + if (file && H5FD_close(file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file") + + FUNC_LEAVE_API(ret_value) +} + +/*------------------------------------------------------------------------- + * Function: H5FD__get_onion_revision_count + * + * Purpose: Private version of H5FDget_onion_revision_count() + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__get_onion_revision_count(H5FD_t *file, uint64_t *revision_count) +{ + uint64_t op_code; + uint64_t flags; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert(file); + HDassert(revision_count); + + op_code = H5FD_CTL_GET_NUM_REVISIONS; + flags = H5FD_CTL_FAIL_IF_UNKNOWN_FLAG; + + /* Get the number of revisions via the ctl callback */ + if (H5FD_ctl(file, op_code, flags, NULL, (void **)&revision_count) < 0) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_write_final_history + * + * Purpose: Write final history to appropriate backing file on file close + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_write_final_history(H5FD_onion_t *file) +{ + size_t size = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + /* TODO: history EOF may not be correct (under what circumstances?) */ + if (0 == (size = H5FD__onion_write_history(&(file->history), file->onion_file, file->onion_eof, + file->onion_eof))) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write final history") + + if (size != file->header.history_size) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "written history differed from expected size") + + /* Is last write operation to history file; no need to extend to page + * boundary if set to page-align. + */ + file->onion_eof += size; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_write_final_history() */ diff --git a/src/H5FDonion.h b/src/H5FDonion.h new file mode 100644 index 0000000..04fd2ff --- /dev/null +++ b/src/H5FDonion.h @@ -0,0 +1,200 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: The public header file for the Onion VFD. + */ +#ifndef H5FDonion_H +#define H5FDonion_H + +#define H5FD_ONION (H5FDperform_init(H5FD_onion_init)) +#define H5FD_ONION_VALUE H5_VFD_ONION + +/* Current version of the fapl info struct */ +#define H5FD_ONION_FAPL_INFO_VERSION_CURR 1 + +/* Flag to open a file that has a locked header (after crashes, for example) */ +#define H5FD_ONION_FAPL_INFO_FLAG_FORCE_OPEN 1 + +/* Flag to enable opening older revisions in write mode, creating a tree */ +#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY 0x1 + +/* Flag to require page alignment of onion revision data */ +#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT 0x2 + +/* Max length of a comment */ +#define H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN 255 + +/* Indicates that you want the latest revision + * TODO: Does this work? + */ +#define H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST UINT64_MAX + +typedef enum H5FD_onion_target_file_constant_t { + H5FD_ONION_STORE_TARGET_H5, /* Onion history as part of HDF5 file */ + H5FD_ONION_STORE_TARGET_ONION, /* Separate, single "onion" file */ +} H5FD_onion_target_file_constant_t; + +/*----------------------------------------------------------------------------- + * Structure H5FD_onion_fapl_info_t + * + * Purpose: Encapsulate info for the Onion driver FAPL entry. + * + * version: Future-proofing identifier. Informs struct membership. + * Must equal H5FD_ONION_FAPL_VERSION_CURR to be considered valid. + * + * backing_fapl_id: + * Backing or 'child' FAPL ID to handle I/O with the + * underlying backing store. If the onion data is stored as a + * separate file, it must use the same backing driver as the + * original file. + * + * page_size: Size of the amended data pages. If opening an existing file, + * must equal the existing page size or zero. If creating a new + * file or an initial revision of an existing file, must be a + * power of 2. + * + * store_target: + * Enumerated/defined value identifying where the history data is + * stored, either in the same file (appended to HDF5 data) or a + * separate file. Other options may be added in later versions. + * + * + H5FD_ONION_FAPL_STORE_MODE_SEPARATE_SINGLE (1) + * Onion history is stored in a single, separate "onion + * file". Shares filename and path as hdf5 file (if any), + * with only a different filename extension. + * + * revision_num: Which revision to open. Must be 0 (the original file) or the + * revision number of an existing revision. + * Revision ID -1 is reserved to open the most recently-created + * revision in history. + * + * force_write_open: + * Flag to ignore the write-lock flag in the onion data + * and attempt to open the file write-only anyway. + * This may be relevant if, for example, the library crashed + * while the file was open in write mode and the write-lock + * flag was not cleared. + * Must equal H5FD_ONION_FAPL_FLAG_FORCE_OPEN to enable. + * + * creation_flags: + * Flag used only when instantiating an Onion file. + * If the relevant bit is set to a nonzero value, its feature + * will be enabled. + * + * + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY + * (1, bit 1) + * User will be allowed to open arbitrary revisions + * in write mode. + * If disabled (0), only the most recent revision may be + * opened for amendment. + * + * + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT (2, bit 2) + * Onion history metadata will align to page_size. + * Partial pages of unused space will occur in the file, + * but may improve read performance from the backing store + * on some systems. + * If disabled (0), padding will not be inserted to align + * to page boundaries. + * + * + <Remaining bits reserved> + * + * comment: User-supplied NULL-terminated comment for a revision to be + * written. + * Cannot be longer than H5FD_ONION_FAPL_COMMENT_MAX_LEN. + * Ignored if part of a FAPL used to open in read mode. + * + * The comment for a revision may be modified prior to committing + * to the revision (closing the file and writing the record) + * with a call to H5FDfctl(). + * This H5FDfctl overwrite may be used to exceed constraints of + * maximum string length and the NULL-terminator requirement. + * + *----------------------------------------------------------------------------- + */ +typedef struct H5FD_onion_fapl_info_t { + uint8_t version; + hid_t backing_fapl_id; + uint32_t page_size; + H5FD_onion_target_file_constant_t store_target; + uint64_t revision_num; + uint8_t force_write_open; + uint8_t creation_flags; + char comment[H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN + 1]; +} H5FD_onion_fapl_info_t; + +#ifdef __cplusplus +extern "C" { +#endif + +H5_DLL hid_t H5FD_onion_init(void); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5P + * + * \brief get the onion info from the file access property list + * + * \param[in] fapl_id The ID of the file access property list + * \param[out] fa_out The pointer to the structure H5FD_onion_fapl_info_t + * + * \return \herr_t + * + * \details H5Pget_fapl_onion() retrieves the structure H5FD_onion_fapl_info_t + * from the file access property list that is set for the onion VFD + * driver. + */ +H5_DLL herr_t H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5P + * + * \brief set the onion info for the file access property list + * + * \param[in] fapl_id The ID of the file access property list + * \param[in] fa The pointer to the structure H5FD_onion_fapl_info_t + * + * \return \herr_t + * + * \details H5Pset_fapl_onion() sets the structure H5FD_onion_fapl_info_t + * for the file access property list that is set for the onion VFD + * driver. + */ +H5_DLL herr_t H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa); + +/** + * -------------------------------------------------------------------------- + * \ingroup H5FD + * + * \brief get the number of revisions + * + * \param[in] filename The name of the onion file + * \param[in] fapl_id The ID of the file access property list + * \param[out] revision_count The number of revisions + * + * \return \herr_t + * + * \details H5FDonion_get_revision_count() returns the number of revisions + * for an onion file. It takes the file name and file access property + * list that is set for the onion VFD driver. + * + */ +H5_DLL herr_t H5FDonion_get_revision_count(const char *filename, hid_t fapl_id, uint64_t *revision_count); + +#ifdef __cplusplus +} +#endif + +#endif /* H5FDonion_H */ diff --git a/src/H5FDonion_header.c b/src/H5FDonion_header.c new file mode 100644 index 0000000..a1d6c28 --- /dev/null +++ b/src/H5FDonion_header.c @@ -0,0 +1,231 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Code for the onion file's header + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDonion.h" /* Onion file driver */ +#include "H5FDonion_priv.h" /* Onion file driver internals */ + +/*----------------------------------------------------------------------------- + * Function: H5FD_ingest_header + * + * Purpose: Read and decode the history header information from `raw_file` + * at `addr`, and store the decoded information in the structure + * at `hdr_out`. + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_ingest_header(H5FD_onion_header_t *hdr_out, H5FD_t *raw_file, haddr_t addr) +{ + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + haddr_t size = (haddr_t)H5FD_ONION_ENCODED_SIZE_HEADER; + uint32_t sum = 0; + + FUNC_ENTER_PACKAGE; + + if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "header indicates history beyond EOF") + + if (NULL == (buf = H5MM_malloc(sizeof(char) * size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space") + + if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA") + + if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read history header from file") + + if (H5FD__onion_header_decode(buf, hdr_out) == 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history header") + + sum = H5_checksum_fletcher32(buf, size - 4); + if (hdr_out->checksum != sum) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored") + +done: + H5MM_xfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_ingest_header() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__onion_write_header + * + * Purpose: Write in-memory history header to appropriate backing file. + * Overwrites existing header data. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_write_header(H5FD_onion_header_t *header, H5FD_t *file) +{ + uint32_t sum = 0; /* Not used, but required by the encoder */ + uint64_t size = 0; + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HEADER))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer for updated history header") + + if (0 == (size = H5FD__onion_header_encode(header, buf, &sum))) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "problem encoding updated history header") + + if (H5FD_write(file, H5FD_MEM_DRAW, 0, (haddr_t)size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "can't write updated history header") + +done: + H5MM_xfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_write_header()*/ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_header_decode + * + * Purpose: Attempt to read a buffer and store it as a history-header + * structure. + * + * Implementation must correspond with + * H5FD__onion_header_encode(). + * + * Return: Success: Number of bytes read from buffer + * Failure: 0 + *----------------------------------------------------------------------------- + */ +size_t +H5FD__onion_header_decode(unsigned char *buf, H5FD_onion_header_t *header) +{ + uint32_t ui32 = 0; + uint32_t sum = 0; + uint64_t ui64 = 0; + uint8_t *ui8p = NULL; + unsigned char *ptr = NULL; + size_t ret_value = 0; + + FUNC_ENTER_PACKAGE; + + HDassert(buf != NULL); + HDassert(header != NULL); + HDassert(H5FD_ONION_HEADER_VERSION_CURR == header->version); + + if (HDstrncmp((const char *)buf, H5FD_ONION_HEADER_SIGNATURE, 4)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header signature") + + if (buf[4] != H5FD_ONION_HEADER_VERSION_CURR) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header version") + + ptr = buf + 5; + ui32 = 0; + HDmemcpy(&ui32, ptr, 3); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, header->flags); + ptr += 3; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, header->page_size); + ptr += 4; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT32DECODE(ui8p, header->origin_eof); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT32DECODE(ui8p, header->history_addr); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT32DECODE(ui8p, header->history_size); + ptr += 8; + + sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, header->checksum); + ptr += 4; + + if (sum != header->checksum) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch") + + ret_value = (size_t)(ptr - buf); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_header_decode() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_header_encode + * + * Purpose: Write history-header structure to the given buffer. + * All multi-byte elements are stored in little-endian word order. + * + * Implementation must correspond with + * H5FD__onion_header_decode(). + * + * The destination buffer must be sufficiently large to hold the + * encoded contents (H5FD_ONION_ENCODED_SIZE_HEADER). + * + * Return: Number of bytes written to buffer. + * The checksum of the generated buffer contents (excluding the + * checksum itself) is stored in the pointer `checksum`). + *----------------------------------------------------------------------------- + */ +size_t +H5FD__onion_header_encode(H5FD_onion_header_t *header, unsigned char *buf, uint32_t *checksum /*out*/) +{ + unsigned char *ptr = buf; + size_t ret_value = 0; + + FUNC_ENTER_PACKAGE_NOERR; + + HDassert(buf != NULL); + HDassert(checksum != NULL); + HDassert(header != NULL); + HDassert(H5FD_ONION_HEADER_VERSION_CURR == header->version); + HDassert(0 == (header->flags & 0xFF000000)); /* max three bits long */ + + HDmemcpy(ptr, H5FD_ONION_HEADER_SIGNATURE, 4); + ptr += 4; + HDmemcpy(ptr, (unsigned char *)&header->version, 1); + ptr += 1; + UINT32ENCODE(ptr, header->flags); + ptr -= 1; /* truncate to three bytes */ + UINT32ENCODE(ptr, header->page_size); + UINT64ENCODE(ptr, header->origin_eof); + UINT64ENCODE(ptr, header->history_addr); + UINT64ENCODE(ptr, header->history_size); + *checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + UINT32ENCODE(ptr, *checksum); + ret_value = (size_t)(ptr - buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_header_encode() */ diff --git a/src/H5FDonion_header.h b/src/H5FDonion_header.h new file mode 100644 index 0000000..cb3941b --- /dev/null +++ b/src/H5FDonion_header.h @@ -0,0 +1,56 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Interface for the onion file's header + */ + +#ifndef H5FDonion_header_H +#define H5FDonion_header_H + +/* Number of bytes to encode fixed-size components */ +#define H5FD_ONION_ENCODED_SIZE_HEADER 40 + +/* Flags must align exactly one per bit, up to 24 bits */ +#define H5FD_ONION_HEADER_FLAG_WRITE_LOCK 0x1 +#define H5FD_ONION_HEADER_FLAG_DIVERGENT_HISTORY 0x2 +#define H5FD_ONION_HEADER_FLAG_PAGE_ALIGNMENT 0x4 +#define H5FD_ONION_HEADER_SIGNATURE "OHDH" +#define H5FD_ONION_HEADER_VERSION_CURR 1 + +/* In-memory representation of the on-store onion history file header. + */ +typedef struct H5FD_onion_header_t { + uint8_t version; + uint32_t flags; /* At most three bytes used! */ + uint32_t page_size; + uint64_t origin_eof; /* Size of the 'original' canonical file */ + uint64_t history_addr; + uint64_t history_size; + uint32_t checksum; +} H5FD_onion_header_t; + +#ifdef __cplusplus +extern "C" { +#endif +H5_DLL herr_t H5FD__onion_ingest_header(H5FD_onion_header_t *hdr_out, H5FD_t *raw_file, haddr_t addr); +H5_DLL herr_t H5FD__onion_write_header(H5FD_onion_header_t *header, H5FD_t *file); +H5_DLL size_t H5FD__onion_header_decode(unsigned char *buf, H5FD_onion_header_t *header); +H5_DLL size_t H5FD__onion_header_encode(H5FD_onion_header_t *header, unsigned char *buf, uint32_t *checksum); + +#ifdef __cplusplus +} +#endif + +#endif /* H5FDonion_header_H */ diff --git a/src/H5FDonion_history.c b/src/H5FDonion_history.c new file mode 100644 index 0000000..501a1f7 --- /dev/null +++ b/src/H5FDonion_history.c @@ -0,0 +1,305 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Code for the onion file's history + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDonion.h" /* Onion file driver */ +#include "H5FDonion_priv.h" /* Onion file driver internals */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_write_history + * + * Purpose: Read and decode the history information from `raw_file` at + * `addr` .. `addr + size` (taken from history header), and store + * the decoded information in the structure at `history_out`. + * + * Returns: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_ingest_history(H5FD_onion_history_t *history_out, H5FD_t *raw_file, haddr_t addr, haddr_t size) +{ + unsigned char *buf = NULL; + uint32_t sum = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + HDassert(history_out); + HDassert(raw_file); + + /* Set early so we can clean up properly on errors */ + history_out->record_locs = NULL; + + if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "header indicates history beyond EOF"); + + if (NULL == (buf = H5MM_malloc(sizeof(char) * size))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate buffer space"); + + if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA"); + + if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read history from file"); + + if (H5FD__onion_history_decode(buf, history_out) != size) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history (initial)"); + + sum = H5_checksum_fletcher32(buf, size - 4); + if (history_out->checksum != sum) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored"); + + if (history_out->n_revisions > 0) + if (NULL == (history_out->record_locs = + H5MM_calloc(history_out->n_revisions * sizeof(H5FD_onion_record_loc_t)))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "can't allocate record pointer list"); + + if (H5FD__onion_history_decode(buf, history_out) != size) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode history (final)"); + +done: + H5MM_xfree(buf); + if (ret_value < 0) + H5MM_xfree(history_out->record_locs); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_ingest_history() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_write_history + * + * Purpose: Encode and write history to file at the given address. + * + * Returns: Success: Number of bytes written to destination file (always non-zero) + * Failure: 0 + *----------------------------------------------------------------------------- + */ +uint64_t +H5FD__onion_write_history(H5FD_onion_history_t *history, H5FD_t *file, haddr_t off_start, + haddr_t filesize_curr) +{ + uint32_t _sum = 0; /* Required by the API call but unused here */ + uint64_t size = 0; + unsigned char *buf = NULL; + uint64_t ret_value = 0; + + FUNC_ENTER_PACKAGE; + + if (NULL == (buf = H5MM_malloc(H5FD_ONION_ENCODED_SIZE_HISTORY + + (H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * history->n_revisions)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, 0, "can't allocate buffer for updated history") + + if (0 == (size = H5FD__onion_history_encode(history, buf, &_sum))) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, 0, "problem encoding updated history") + + if ((size + off_start > filesize_curr) && (H5FD_set_eoa(file, H5FD_MEM_DRAW, off_start + size) < 0)) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, 0, "can't modify EOA for updated history") + + if (H5FD_write(file, H5FD_MEM_DRAW, off_start, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, 0, "can't write history as intended") + + ret_value = size; + +done: + H5MM_xfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_write_history() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_history_decode + * + * Purpose: Attempt to read a buffer and store it as a history + * structure. + * + * Implementation must correspond with + * H5FD__onion_history_encode(). + * + * MUST BE CALLED TWICE: + * On the first call, n_records in the destination structure must + * be zero, and record_locs be NULL. + * + * If the buffer is well-formed, the destination structure is + * tentatively populated with fixed-size values, and the number of + * bytes read are returned. + * + * Prior to the second call, the user must allocate space for + * record_locs to hold n_records record-pointer structs. + * + * Then the decode operation is called a second time, and all + * components will be populated (and again number of bytes read is + * returned). + * + * Return: Success: Number of bytes read from buffer + * Failure: 0 + *----------------------------------------------------------------------------- + */ +size_t +H5FD__onion_history_decode(unsigned char *buf, H5FD_onion_history_t *history) +{ + uint32_t ui32 = 0; + uint32_t sum = 0; + uint64_t ui64 = 0; + uint64_t n_revisions = 0; + uint8_t *ui8p = NULL; + unsigned char *ptr = NULL; + size_t ret_value = 0; + + FUNC_ENTER_PACKAGE; + + HDassert(buf != NULL); + HDassert(history != NULL); + HDassert(H5FD_ONION_HISTORY_VERSION_CURR == history->version); + + if (HDstrncmp((const char *)buf, H5FD_ONION_HISTORY_SIGNATURE, 4)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature") + + if (H5FD_ONION_HISTORY_VERSION_CURR != buf[4]) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid version") + + ptr = buf + 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, n_revisions); + ptr += 8; + + if (0 == history->n_revisions) { + history->n_revisions = n_revisions; + ptr += H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * n_revisions; + } + else { + if (history->n_revisions != n_revisions) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "history argument suggests different revision count than encoded buffer") + if (NULL == history->record_locs) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "list is NULL -- cannot populate") + + for (uint64_t i = 0; i < n_revisions; i++) { + H5FD_onion_record_loc_t *rloc = &history->record_locs[i]; + + /* Decode into appropriately sized types, then do a checked + * assignment to the struct value. We don't have access to + * the H5F_t struct for this file, so we can't use the + * offset/length macros in H5Fprivate.h. + */ + uint64_t record_size; + uint64_t phys_addr; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, phys_addr); + H5_CHECKED_ASSIGN(rloc->phys_addr, haddr_t, phys_addr, uint64_t); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, record_size); + H5_CHECKED_ASSIGN(rloc->record_size, hsize_t, record_size, uint64_t); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, rloc->checksum); + ptr += 4; + } + } + + sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, history->checksum); + ptr += 4; + + if (sum != history->checksum) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch") + + ret_value = (size_t)(ptr - buf); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_history_decode() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_history_encode + * + * Purpose: Write history structure to the given buffer. + * All multi-byte elements are stored in little-endian word order. + * + * Implementation must correspond with + * H5FD__onion_history_decode(). + * + * The destination buffer must be sufficiently large to hold the + * encoded contents. + * (Hint: `sizeof(history struct) + + * sizeof(record-pointer-struct) * n_records)` guarantees + * ample/excess space.) + * + * Return: Number of bytes written to buffer. + * The checksum of the generated buffer contents (excluding the + * checksum itself) is stored in the pointer `checksum`). + *----------------------------------------------------------------------------- + */ +size_t +H5FD__onion_history_encode(H5FD_onion_history_t *history, unsigned char *buf, uint32_t *checksum) +{ + unsigned char *ptr = buf; + size_t vers_u32 = (uint32_t)history->version; /* pad out unused bytes */ + + FUNC_ENTER_PACKAGE_NOERR; + + HDassert(history != NULL); + HDassert(H5FD_ONION_HISTORY_VERSION_CURR == history->version); + HDassert(buf != NULL); + HDassert(checksum != NULL); + + HDmemcpy(ptr, H5FD_ONION_HISTORY_SIGNATURE, 4); + ptr += 4; + UINT32ENCODE(ptr, vers_u32); + UINT64ENCODE(ptr, history->n_revisions); + if (history->n_revisions > 0) { + HDassert(history->record_locs != NULL); + for (uint64_t i = 0; i < history->n_revisions; i++) { + H5FD_onion_record_loc_t *rloc = &history->record_locs[i]; + + /* Do a checked assignment from the struct value into appropriately + * sized types. We don't have access to the H5F_t struct for this + * file, so we can't use the offset/length macros in H5Fprivate.h. + */ + uint64_t phys_addr; + uint64_t record_size; + + H5_CHECKED_ASSIGN(phys_addr, uint64_t, rloc->phys_addr, haddr_t); + H5_CHECKED_ASSIGN(record_size, uint64_t, rloc->record_size, hsize_t); + + UINT64ENCODE(ptr, phys_addr); + UINT64ENCODE(ptr, record_size); + UINT32ENCODE(ptr, rloc->checksum); + } + } + *checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + UINT32ENCODE(ptr, *checksum); + + FUNC_LEAVE_NOAPI((size_t)(ptr - buf)); +} /* end H5FD__onion_history_encode() */ diff --git a/src/H5FDonion_history.h b/src/H5FDonion_history.h new file mode 100644 index 0000000..bf27e6a --- /dev/null +++ b/src/H5FDonion_history.h @@ -0,0 +1,63 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Interface for the onion file's history + */ + +#ifndef H5FDonion_history_H +#define H5FDonion_history_H + +/* Number of bytes to encode fixed-size components */ +#define H5FD_ONION_ENCODED_SIZE_HISTORY 20 + +#define H5FD_ONION_HISTORY_SIGNATURE "OWHS" +#define H5FD_ONION_HISTORY_VERSION_CURR 1 + +/* In-memory representation of the on-store revision record. + * Used in the history. + */ +typedef struct H5FD_onion_record_loc_t { + haddr_t phys_addr; + hsize_t record_size; + uint32_t checksum; +} H5FD_onion_record_loc_t; + +/* In-memory representation of the on-store history record/summary. + */ +typedef struct H5FD_onion_history_t { + uint8_t version; + uint64_t n_revisions; + H5FD_onion_record_loc_t *record_locs; + uint32_t checksum; +} H5FD_onion_history_t; + +#ifdef __cplusplus +extern "C" { +#endif +H5_DLL herr_t H5FD__onion_ingest_history(H5FD_onion_history_t *history_out, H5FD_t *raw_file, haddr_t addr, + haddr_t size); + +H5_DLL uint64_t H5FD__onion_write_history(H5FD_onion_history_t *history, H5FD_t *file, haddr_t off_start, + haddr_t filesize_curr); + +H5_DLL size_t H5FD__onion_history_decode(unsigned char *buf, H5FD_onion_history_t *history); +H5_DLL size_t H5FD__onion_history_encode(H5FD_onion_history_t *history, unsigned char *buf, + uint32_t *checksum); + +#ifdef __cplusplus +} +#endif + +#endif /* H5FDonion_history_H */ diff --git a/src/H5FDonion_index.c b/src/H5FDonion_index.c new file mode 100644 index 0000000..90eaf0e --- /dev/null +++ b/src/H5FDonion_index.c @@ -0,0 +1,935 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Code for the archival and revision indexes + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDonion.h" /* Onion file driver */ +#include "H5FDonion_priv.h" /* Onion file driver internals */ + +/* 2^n for uint64_t types -- H5_EXP2 unsafe past 32 bits */ +#define U64_EXP2(n) ((uint64_t)1 << (n)) + +static int H5FD__onion_archival_index_list_sort_cmp(const void *, const void *); +static herr_t H5FD__onion_revision_index_resize(H5FD_onion_revision_index_t *rix); + +/*----------------------------------------------------------------------------- + * Read and decode the revision_record information from `raw_file` at + * `addr` .. `addr + size` (taken from history), and store the decoded + * information in the structure at `r_out`. + *----------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_ingest_revision_record(H5FD_onion_revision_record_t *r_out, H5FD_t *raw_file, + const H5FD_onion_history_t *history, uint64_t revision_num) +{ + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + uint64_t n = 0; + uint64_t high = 0; + uint64_t low = 0; + uint64_t range = 0; + uint32_t sum = 0; + haddr_t addr = 0; + size_t size = 0; + + FUNC_ENTER_PACKAGE; + + HDassert(r_out); + HDassert(raw_file); + HDassert(history); + HDassert(history->record_locs); + HDassert(history->n_revisions > 0); + + high = history->n_revisions - 1; + range = high; + addr = history->record_locs[high].phys_addr; + size = history->record_locs[high].record_size; + + /* Initialize r_out + * + * TODO: This function should completely initialize r_out. Relying on + * other code to some of the work while we just paste over parts + * of the struct here is completely bananas. + */ + r_out->comment = H5MM_xfree(r_out->comment); + r_out->archival_index.list = H5MM_xfree(r_out->archival_index.list); + + if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "at least one record extends beyond EOF") + + /* recovery-open may have EOA below revision record */ + if ((H5FD_get_eoa(raw_file, H5FD_MEM_DRAW) < (addr + size)) && + (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)) { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't modify EOA"); + } + + /* Perform binary search on records to find target revision by ID. + * As IDs are added sequentially, they are guaranteed to be sorted. + */ + while (range > 0) { + n = (range / 2) + low; + addr = history->record_locs[n].phys_addr; + size = history->record_locs[n].record_size; + + if (NULL == (buf = H5MM_malloc(sizeof(char) * size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space") + + if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read revision record from file") + + if (H5FD__onion_revision_record_decode(buf, r_out) != size) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (initial)") + + sum = H5_checksum_fletcher32(buf, size - 4); + if (r_out->checksum != sum) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored") + + if (revision_num == r_out->revision_num) + break; + + H5MM_xfree(buf); + buf = NULL; + + r_out->archival_index.n_entries = 0; + r_out->comment_size = 0; + + if (r_out->revision_num < revision_num) + low = (n == high) ? high : n + 1; + else + high = (n == low) ? low : n - 1; + range = high - low; + } /* end while 'non-leaf' binary search */ + + if (range == 0) { + n = low; + addr = history->record_locs[n].phys_addr; + size = history->record_locs[n].record_size; + + if (NULL == (buf = H5MM_malloc(sizeof(char) * size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer space") + + if (H5FD_read(raw_file, H5FD_MEM_DRAW, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't read revision record from file") + + if (H5FD__onion_revision_record_decode(buf, r_out) != size) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (initial)") + + sum = H5_checksum_fletcher32(buf, size - 4); + if (r_out->checksum != sum) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "checksum mismatch between buffer and stored") + + if (revision_num != r_out->revision_num) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, + "could not find target revision!") /* TODO: corrupted? */ + } /* end if revision ID at 'leaf' in binary search */ + + if (r_out->comment_size > 0) + if (NULL == (r_out->comment = H5MM_malloc(sizeof(char) * r_out->comment_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate comment space") + + if (r_out->archival_index.n_entries > 0) + if (NULL == (r_out->archival_index.list = + H5MM_calloc(r_out->archival_index.n_entries * sizeof(H5FD_onion_index_entry_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate index entry list") + + if (H5FD__onion_revision_record_decode(buf, r_out) != size) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "can't decode revision record (final)") + +done: + H5MM_xfree(buf); + if (ret_value == FAIL) { + H5MM_xfree(r_out->comment); + H5MM_xfree(r_out->archival_index.list); + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_ingest_revision_record() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_archival_index_is_valid + * + * Purpose: Determine whether an archival index structure is valid. + * + * + Verify page size (power of two). + * + Verify list exists. + * + Verify list contents: + * + Sorted by increasing logical address (no duplicates) + * + Logical addresses are multiples of page size. + * + * Return: TRUE/FALSE + *----------------------------------------------------------------------------- + */ +hbool_t +H5FD__onion_archival_index_is_valid(const H5FD_onion_archival_index_t *aix) +{ + hbool_t ret_value = TRUE; + + FUNC_ENTER_PACKAGE_NOERR; + + HDassert(aix); + + if (H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR != aix->version) + HGOTO_DONE(FALSE) + if (NULL == aix->list) + HGOTO_DONE(FALSE) + + /* Ensure list is sorted on logical_page field */ + if (aix->n_entries > 1) + for (uint64_t i = 1; i < aix->n_entries - 1; i++) + if (aix->list[i + 1].logical_page <= aix->list[i].logical_page) + HGOTO_DONE(FALSE) + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_archival_index_is_valid() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_archival_index_find + * + * Purpose: Retrieve the archival index entry by logical page ID. + * + * The archival index pointer must point to a valid index entry. + * The entry out pointer-pointer cannot be null. + * + * Return: Success: Positive value (1) -- entry found. + * Entry out pointer-pointer is set to point to entry. + * Failure: Zero (0) -- entry not found. + * Entry out pointer-pointer is unmodified. + *----------------------------------------------------------------------------- + */ +int +H5FD__onion_archival_index_find(const H5FD_onion_archival_index_t *aix, uint64_t logical_page, + const H5FD_onion_index_entry_t **entry_out) +{ + uint64_t low = 0; + uint64_t high = 0; + uint64_t n = 0; + uint64_t range = 0; + H5FD_onion_index_entry_t *x = NULL; + int ret_value = 0; + + FUNC_ENTER_PACKAGE_NOERR; + + HDassert(aix); + HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == aix->version); + HDassert(entry_out); + if (aix->n_entries != 0) + HDassert(aix->list); + + high = aix->n_entries - 1; + range = high; + + /* Trivial cases */ + if (aix->n_entries == 0 || logical_page > aix->list[high].logical_page || + logical_page < aix->list[0].logical_page) + HGOTO_DONE(0) + + /* + * Binary search on sorted list + */ + + /* Winnow down to first of found or one element */ + while (range > 0) { + HDassert(high < aix->n_entries); + n = low + (range / 2); + x = &(aix->list[n]); + if (x->logical_page == logical_page) { + *entry_out = x; /* element found at fence */ + ret_value = 1; + goto done; + } + else if (x->logical_page < logical_page) { + low = (n == high) ? high : n + 1; + } + else { + high = (n == low) ? low : n - 1; + } + range = high - low; + } + + HDassert(high == low); /* one element */ + + /* n == low/high check because we may have tested it already above */ + if ((n != low || n != high) && (aix->list[low].logical_page == logical_page)) { + *entry_out = &aix->list[low]; + ret_value = 1; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_archival_index_find() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_revision_index_destroy + * + * Purpose: Release all resources of a revision index. + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_revision_index_destroy(H5FD_onion_revision_index_t *rix) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR; + + HDassert(rix); + HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version); + + for (size_t i = 0; 0 < rix->_hash_table_n_keys_populated && i < rix->_hash_table_size; i++) { + H5FD_onion_revision_index_hash_chain_node_t *next = NULL; + H5FD_onion_revision_index_hash_chain_node_t *node = rix->_hash_table[i]; + + if (node != NULL) + rix->_hash_table_n_keys_populated -= 1; + + while (node != NULL) { + HDassert(H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR == node->version); + + next = node->next; + H5MM_xfree(node); + node = next; + } + } + H5MM_xfree(rix->_hash_table); + H5MM_xfree(rix); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_revision_index_destroy() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_revision_index_init + * + * Purpose: Initialize a revision index structure with a default starting + * size. A new structure is allocated and populated with initial + * values. + * + * Return: Success: Pointer to newly-allocated structure + * Failure: NULL + *----------------------------------------------------------------------------- + */ +H5FD_onion_revision_index_t * +H5FD__onion_revision_index_init(uint32_t page_size) +{ + uint64_t table_size = U64_EXP2(H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2); + H5FD_onion_revision_index_t *rix = NULL; + H5FD_onion_revision_index_t *ret_value = NULL; + + FUNC_ENTER_PACKAGE; + + HDassert(0 != page_size); + HDassert(POWER_OF_TWO(page_size)); + + if (NULL == (rix = H5MM_calloc(sizeof(H5FD_onion_revision_index_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "cannot allocate index") + + if (NULL == + (rix->_hash_table = H5MM_calloc(table_size * sizeof(H5FD_onion_revision_index_hash_chain_node_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "cannot allocate hash table") + + rix->version = H5FD_ONION_REVISION_INDEX_VERSION_CURR; + rix->n_entries = 0; + /* Compute and store log2(page_size) */ + for (rix->page_size_log2 = 0; (((uint32_t)1 << rix->page_size_log2) & page_size) == 0; + rix->page_size_log2++) + ; + rix->_hash_table_size = table_size; + rix->_hash_table_size_log2 = H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2; + rix->_hash_table_n_keys_populated = 0; + + ret_value = rix; + +done: + if (NULL == ret_value) + H5MM_xfree(rix); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_revision_index_init() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_revision_index_resize() + * + * Purpose: Replace the hash table in the revision index. + * + * Doubles the available number of keys, re-hashes table contents, + * and updates relevant components in the index structure. + * + * Fails if unable to allocate space for larger hash table. + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_revision_index_resize(H5FD_onion_revision_index_t *rix) +{ + H5FD_onion_revision_index_hash_chain_node_t **new_table = NULL; + + uint64_t new_size_log2 = rix->_hash_table_size_log2 + 1; + uint64_t new_size = U64_EXP2(new_size_log2); + uint64_t new_n_keys_populated = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + HDassert(rix); + HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version); + HDassert(rix->_hash_table); + + if (NULL == (new_table = H5MM_calloc(new_size * sizeof(H5FD_onion_revision_index_hash_chain_node_t *)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate new hash table") + + for (uint64_t i = 0; i < rix->_hash_table_size; i++) { + while (rix->_hash_table[i] != NULL) { + H5FD_onion_revision_index_hash_chain_node_t *node = NULL; + uint64_t key = 0; + + /* Pop entry off of bucket stack and re-hash */ + node = rix->_hash_table[i]; + rix->_hash_table[i] = node->next; + node->next = NULL; + key = node->entry_data.logical_page & (new_size - 1); + + if (NULL == new_table[key]) { + new_table[key] = node; + new_n_keys_populated++; + } + else { + node->next = new_table[i]; + new_table[i] = node; + } + } + } + + H5MM_xfree(rix->_hash_table); + rix->_hash_table_size = new_size; + rix->_hash_table_size_log2 = new_size_log2; + rix->_hash_table_n_keys_populated = new_n_keys_populated; + rix->_hash_table = new_table; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_revision_index_resize() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_revision_index_insert() + * + * Purpose: Add an entry to the revision index, or update an existing + * entry. Must be used to update entries as well as add -- + * checksum value will change. + * + * Entry data is copied into separate memory region; user pointer + * can be safley re-used or discarded after operation. + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_revision_index_insert(H5FD_onion_revision_index_t *rix, const H5FD_onion_index_entry_t *entry) +{ + uint64_t key = 0; + H5FD_onion_revision_index_hash_chain_node_t *node = NULL; + H5FD_onion_revision_index_hash_chain_node_t **append_dest = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + HDassert(rix); + HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version); + HDassert(entry); + + /* Resize and re-hash table if necessary */ + if (rix->n_entries >= (rix->_hash_table_size * 2) || + rix->_hash_table_n_keys_populated >= (rix->_hash_table_size / 2)) { + if (H5FD__onion_revision_index_resize(rix) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_NONE_MINOR, FAIL, "unable to resize and hash table") + } + + key = entry->logical_page & (rix->_hash_table_size - 1); + HDassert(key < rix->_hash_table_size); + + if (NULL == rix->_hash_table[key]) { + /* Key maps to empty bucket */ + + append_dest = &rix->_hash_table[key]; + rix->_hash_table_n_keys_populated++; + } + else { + /* Key maps to populated bucket */ + + for (node = rix->_hash_table[key]; node != NULL; node = node->next) { + append_dest = &node->next; /* look for bucket tail */ + if (entry->logical_page == node->entry_data.logical_page) { + if (entry->phys_addr != node->entry_data.phys_addr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "physical address mismatch"); + } + HDmemcpy(&node->entry_data, entry, sizeof(H5FD_onion_index_entry_t)); + append_dest = NULL; /* Node updated, do not append */ + break; + } + } + } + + /* Add new entry to bucket chain */ + if (append_dest != NULL) { + if (NULL == (node = H5MM_malloc(sizeof(H5FD_onion_revision_index_hash_chain_node_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "cannot allocate new ash chain node") + node->version = H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR; + node->next = NULL; + HDmemcpy(&node->entry_data, entry, sizeof(H5FD_onion_index_entry_t)); + *append_dest = node; + rix->n_entries++; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_revision_index_insert() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_revision_index_find() + * + * + * Purpose: Get pointer to revision index entry with the given page number, + * if it exists in the index. + * + * Return: Success: Positive value (1) -- entry found. + * Entry out pointer-pointer is set to point to entry. + * Failure: Zero (0) -- entry not found. + * Entry out pointer-pointer is unmodified. + *----------------------------------------------------------------------------- + */ +int +H5FD__onion_revision_index_find(const H5FD_onion_revision_index_t *rix, uint64_t logical_page, + const H5FD_onion_index_entry_t **entry_out) +{ + uint64_t key = 0; + int ret_value = 0; + + FUNC_ENTER_PACKAGE_NOERR; + + HDassert(rix); + HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version); + HDassert(rix->_hash_table); + HDassert(entry_out); + + key = logical_page & (rix->_hash_table_size - 1); + HDassert(key < rix->_hash_table_size); + + if (rix->_hash_table[key] != NULL) { + H5FD_onion_revision_index_hash_chain_node_t *node = NULL; + + for (node = rix->_hash_table[key]; node != NULL; node = node->next) { + if (logical_page == node->entry_data.logical_page) { + *entry_out = &node->entry_data; + ret_value = 1; + break; + } + } + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_revision_index_find() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_revision_record_decode + * + * Purpose: Attempt to read a buffer and store it as a revision record + * structure. + * + * Implementation must correspond with + * H5FD__onion_revision_record_encode(). + * + * MUST BE CALLED TWICE: + * On the first call, n_entries and comment_size in the + * destination structure must all all be zero, and their + * respective variable-length components (index_entry_list, + * comment) must all be NULL. + * + * If the buffer is well-formed, the destination structure is + * tentatively populated with fixed-size values, and the number of + * bytes read are returned. + * + * Prior to the second call, the user must allocate space for the + * variable-length components, in accordance with the associated + * indicators (array of index-entry structures for + * index_entry_list, of size n_entries; character arrays for + * comment, allocated with the *_size number of bytes -- space + * for NULL-terminator is included in _size). + * + * Then the decode operation is called a second time, and all + * components will be populated (and again number of bytes read is + * returned). + * + * Return: Success: Number of bytes read from buffer + * Failure: 0 + *----------------------------------------------------------------------------- + */ +size_t +H5FD__onion_revision_record_decode(unsigned char *buf, H5FD_onion_revision_record_t *record) +{ + uint32_t ui32 = 0; + uint32_t page_size = 0; + uint32_t sum = 0; + uint64_t ui64 = 0; + uint64_t n_entries = 0; + uint32_t comment_size = 0; + uint8_t *ui8p = NULL; + unsigned char *ptr = NULL; + size_t ret_value = 0; + + FUNC_ENTER_PACKAGE; + + HDassert(buf != NULL); + HDassert(record != NULL); + HDassert(H5FD_ONION_REVISION_RECORD_VERSION_CURR == record->version); + HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version); + + if (HDstrncmp((const char *)buf, H5FD_ONION_REVISION_RECORD_SIGNATURE, 4)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature") + + if (H5FD_ONION_REVISION_RECORD_VERSION_CURR != buf[4]) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid record version") + + ptr = buf + 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, record->revision_num); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, record->parent_revision_num); + ptr += 8; + + HDmemcpy(record->time_of_creation, ptr, 16); + ptr += 16; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, record->logical_eof); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, page_size); + ptr += 4; + + if (page_size == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size is zero") + if (!POWER_OF_TWO(page_size)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size not power of two") + + for (record->archival_index.page_size_log2 = 0; + (((uint32_t)1 << record->archival_index.page_size_log2) & page_size) == 0; + record->archival_index.page_size_log2++) + ; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, n_entries); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, comment_size); + ptr += 4; + + if (record->archival_index.n_entries == 0) { + record->archival_index.n_entries = n_entries; + ptr += H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * n_entries; + } + else if (n_entries != record->archival_index.n_entries) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "n_entries in archival index does not match decoded") + } + else { + H5FD_onion_index_entry_t *entry = NULL; + + if (record->archival_index.list == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no archival index entry list") + + for (size_t i = 0; i < n_entries; i++) { + entry = &record->archival_index.list[i]; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, entry->logical_page); + ptr += 8; + + /* logical_page actually encoded as address; check and convert */ + if (entry->logical_page & (page_size - 1)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "logical address does not align with page size") + + entry->logical_page = entry->logical_page >> record->archival_index.page_size_log2; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, entry->phys_addr); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, sum); + ptr += 4; + + ui32 = H5_checksum_fletcher32((ptr - 20), 16); + if (ui32 != sum) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "index entry checksum mismatch") + } + } + + if (record->comment_size == 0) { + if (record->comment != NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "comment pointer prematurely allocated") + record->comment_size = comment_size; + } + else { + if (record->comment == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no comment pointer") + HDmemcpy(record->comment, ptr, comment_size); + } + ptr += comment_size; + + sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, record->checksum); + ptr += 4; + + if (sum != record->checksum) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch") + + ret_value = (size_t)(ptr - buf); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_revision_record_decode() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_revision_record_encode + * + * Purpose: Write revision-record structure to the given buffer. + * All multi-byte elements are stored in little-endian word order. + * + * Implementation must correspond with + * H5FD__onion_revision_record_decode(). + * + * The destination buffer must be sufficiently large to hold the + * encoded contents. + * (Hint: `sizeof(revision-record-struct) + comment-size + + * sizeof(index-entry-struct) * n_entries)` + * guarantees ample/excess space.) + * + * Return: Number of bytes written to buffer. + * The checksum of the generated buffer contents (excluding the + * checksum itself) is stored in the pointer `checksum`). + *----------------------------------------------------------------------------- + */ +size_t +H5FD__onion_revision_record_encode(H5FD_onion_revision_record_t *record, unsigned char *buf, + uint32_t *checksum) +{ + unsigned char *ptr = buf; /* original pointer */ + uint32_t vers_u32 = (uint32_t)record->version; /* pad out unused bytes */ + uint32_t page_size = 0; + + FUNC_ENTER_PACKAGE_NOERR; + + HDassert(checksum != NULL); + HDassert(buf != NULL); + HDassert(record != NULL); + HDassert(vers_u32 < 0x100); + HDassert(H5FD_ONION_REVISION_RECORD_VERSION_CURR == record->version); + HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version); + + page_size = (uint32_t)(1 << record->archival_index.page_size_log2); + + HDmemcpy(ptr, H5FD_ONION_REVISION_RECORD_SIGNATURE, 4); + ptr += 4; + UINT32ENCODE(ptr, vers_u32); + UINT64ENCODE(ptr, record->revision_num); + UINT64ENCODE(ptr, record->parent_revision_num); + HDmemcpy(ptr, record->time_of_creation, 16); + ptr += 16; + UINT64ENCODE(ptr, record->logical_eof); + UINT32ENCODE(ptr, page_size); + UINT64ENCODE(ptr, record->archival_index.n_entries); + UINT32ENCODE(ptr, record->comment_size); + + if (record->archival_index.n_entries > 0) { + uint64_t page_size_log2 = record->archival_index.page_size_log2; + + HDassert(record->archival_index.list != NULL); + for (uint64_t i = 0; i < record->archival_index.n_entries; i++) { + uint32_t sum = 0; + H5FD_onion_index_entry_t *entry = NULL; + uint64_t logi_addr = 0; + + entry = &record->archival_index.list[i]; + logi_addr = entry->logical_page << page_size_log2; + + UINT64ENCODE(ptr, logi_addr); + UINT64ENCODE(ptr, entry->phys_addr); + sum = H5_checksum_fletcher32((ptr - 16), 16); + UINT32ENCODE(ptr, sum); + } + } + + if (record->comment_size > 0) { + HDassert(record->comment != NULL && *record->comment != '\0'); + HDmemcpy(ptr, record->comment, record->comment_size); + ptr += record->comment_size; + } + + *checksum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + UINT32ENCODE(ptr, *checksum); + + FUNC_LEAVE_NOAPI((size_t)(ptr - buf)); +} /* end H5FD__onion_revision_record_encode() */ + +/*----------------------------------------------------------------------------- + * Callback for comparisons in sorting archival index entries by logical_page. + *----------------------------------------------------------------------------- + */ +static int +H5FD__onion_archival_index_list_sort_cmp(const void *_a, const void *_b) +{ + const H5FD_onion_index_entry_t *a = (const H5FD_onion_index_entry_t *)_a; + const H5FD_onion_index_entry_t *b = (const H5FD_onion_index_entry_t *)_b; + + if (a->logical_page < b->logical_page) + return -1; + else if (a->logical_page > b->logical_page) + return 1; + return 0; +} /* end H5FD__onion_archival_index_list_sort_cmp() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_merge_revision_index_into_archival_index + * + * Purpose: Merge index entries from revision index into archival index. + * + * If successful, the archival index is expanded 'behind the + * scenes' and new entries from the revision index are inserted. + * The archival index remains sorted in ascending order of logical + * address. + * + * The conversion to archival index changes logical pages in + * revision index entries to their logical addresses in-file. + * + * Return: SUCCEED/FAIL + *----------------------------------------------------------------------------- + */ +herr_t +H5FD__onion_merge_revision_index_into_archival_index(const H5FD_onion_revision_index_t *rix, + H5FD_onion_archival_index_t *aix) +{ + uint64_t n_kept = 0; + H5FD_onion_index_entry_t *kept_list = NULL; + H5FD_onion_archival_index_t new_aix = { + H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, 0, /* page_size_log2 tbd */ + 0, /* n_entries */ + NULL, /* list pointer (allocated later) */ + }; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE; + + HDassert(rix); + HDassert(aix); + HDassert(H5FD_ONION_REVISION_INDEX_VERSION_CURR == rix->version); + HDassert(H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR == aix->version); + HDassert(aix->page_size_log2 == rix->page_size_log2); + + /* If the revision index is empty there is nothing to archive */ + if (rix->n_entries == 0) + goto done; + + /* Add all revision index entries to new archival list */ + new_aix.page_size_log2 = aix->page_size_log2; + + if (NULL == (new_aix.list = H5MM_calloc(rix->n_entries * sizeof(H5FD_onion_index_entry_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate new archival index list") + + for (uint64_t i = 0; i < rix->_hash_table_size; i++) { + const H5FD_onion_revision_index_hash_chain_node_t *node = NULL; + + for (node = rix->_hash_table[i]; node != NULL; node = node->next) { + HDmemcpy(&new_aix.list[new_aix.n_entries], &node->entry_data, sizeof(H5FD_onion_index_entry_t)); + new_aix.n_entries++; + } + } + + /* Sort the new archival list */ + HDqsort(new_aix.list, new_aix.n_entries, sizeof(H5FD_onion_index_entry_t), + H5FD__onion_archival_index_list_sort_cmp); + + /* Add the old archival index entries to a 'kept' list containing the + * old archival list entries that are not also included in the revision + * list. + * + * Note that kept_list will be NULL if there are no entries in the passed-in + * archival list. + */ + if (aix->n_entries > 0) + if (NULL == (kept_list = H5MM_calloc(aix->n_entries * sizeof(H5FD_onion_index_entry_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate larger archival index list") + + for (uint64_t i = 0; i < aix->n_entries; i++) { + const H5FD_onion_index_entry_t *entry = NULL; + + /* Add only if page not already added from revision index */ + if (H5FD__onion_archival_index_find(&new_aix, aix->list[i].logical_page, &entry) == 0) { + HDmemcpy(&kept_list[n_kept], &aix->list[i], sizeof(H5FD_onion_index_entry_t)); + n_kept++; + } + } + + /* Destroy the old archival list and replace with a list big enough to hold + * the revision list entries and the kept list entries + */ + H5MM_xfree(aix->list); + if (NULL == (aix->list = H5MM_calloc((new_aix.n_entries + n_kept) * sizeof(H5FD_onion_index_entry_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate exact-size archival index list") + + /* Copy (new) revision list entries to replacement list */ + HDmemcpy(aix->list, new_aix.list, sizeof(H5FD_onion_index_entry_t) * new_aix.n_entries); + aix->n_entries = new_aix.n_entries; + + /* Copy (old) kept archival list entries to replacement list */ + if (n_kept > 0) { + HDmemcpy(&aix->list[aix->n_entries], kept_list, sizeof(H5FD_onion_index_entry_t) * n_kept); + aix->n_entries += n_kept; + } + + /* Sort this list */ + HDqsort(aix->list, aix->n_entries, sizeof(H5FD_onion_index_entry_t), + H5FD__onion_archival_index_list_sort_cmp); + +done: + /* Free the temporary lists */ + H5MM_xfree(kept_list); + H5MM_xfree(new_aix.list); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_merge_revision_index_into_archival_index() */ diff --git a/src/H5FDonion_index.h b/src/H5FDonion_index.h new file mode 100644 index 0000000..16e461b --- /dev/null +++ b/src/H5FDonion_index.h @@ -0,0 +1,150 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef H5FDonion_index_H +#define H5FDonion_index_H + +#define H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR 1 + +/* Number of bytes to encode fixed-size components */ +#define H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY 20 +#define H5FD_ONION_ENCODED_SIZE_RECORD_POINTER 20 +#define H5FD_ONION_ENCODED_SIZE_REVISION_RECORD 68 + +#define H5FD_ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR 1 +#define H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2 10 /* 2^n slots */ +#define H5FD_ONION_REVISION_INDEX_VERSION_CURR 1 + +#define H5FD_ONION_REVISION_RECORD_SIGNATURE "ORRS" +#define H5FD_ONION_REVISION_RECORD_VERSION_CURR 1 + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Interface for the archival and revision indexes + */ + +/*----------------------------------------------------------------------------- + * + * Structure H5FD__onion_index_entry + * + * Purpose: Map a page in the logical file to a 'physical address' in the + * onion file. + * + * logical_page: + * + * Page 'id' in the logical file. + * + * phys_addr: + * + * Address/offset of start of page in the onion file. + * + *----------------------------------------------------------------------------- + */ +typedef struct H5FD_onion_index_entry_t { + uint64_t logical_page; + haddr_t phys_addr; +} H5FD_onion_index_entry_t; + +/*----------------------------------------------------------------------------- + * + * Structure H5FD__onion_archival_index + * + * Purpose: Encapsulate archival index and associated data. + * Convenience structure with sanity-checking components. + * + * version: Future-proofing identifier. Informs struct membership. + * Must equal H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR to be + * considered valid. + * + * page_size: Interval to which the `logical_page` component of each list + * entry must align. + * Value is taken from the onion history data; must not change + * following onionization or file or creation of onion file. + * + * n_entries: Number of entries in the list. + * + * list: Pointer to array of archival index entries. + * Cannot be NULL. + * Entries must be sorted by `logical_page_id` in ascending order. + * + *----------------------------------------------------------------------------- + */ +typedef struct H5FD_onion_archival_index_t { + uint8_t version; + uint32_t page_size_log2; + uint64_t n_entries; + H5FD_onion_index_entry_t *list; +} H5FD_onion_archival_index_t; + +/* data structure for storing index entries at a hash key collision */ +/* version 1 implements a singly-linked list */ +typedef struct H5FD_onion_revision_index_hash_chain_node_t H5FD_onion_revision_index_hash_chain_node_t; +struct H5FD_onion_revision_index_hash_chain_node_t { + uint8_t version; + H5FD_onion_index_entry_t entry_data; + H5FD_onion_revision_index_hash_chain_node_t *next; +}; + +typedef struct H5FD_onion_revision_index_t { + uint8_t version; + uint32_t page_size_log2; + uint64_t n_entries; /* count of all entries in table */ + uint64_t _hash_table_size; /* 'slots' in hash table */ + uint64_t _hash_table_size_log2; /* 2^(n) -> 'slots' in hash table */ + uint64_t _hash_table_n_keys_populated; /* count of slots not NULL */ + H5FD_onion_revision_index_hash_chain_node_t **_hash_table; +} H5FD_onion_revision_index_t; + +/* In-memory representation of the on-store revision record. + */ +typedef struct H5FD_onion_revision_record_t { + uint8_t version; + uint64_t revision_num; + uint64_t parent_revision_num; + char time_of_creation[16]; + uint64_t logical_eof; + H5FD_onion_archival_index_t archival_index; + uint32_t comment_size; + char *comment; + uint32_t checksum; +} H5FD_onion_revision_record_t; + +#ifdef __cplusplus +extern "C" { +#endif +H5_DLL herr_t H5FD__onion_ingest_revision_record(H5FD_onion_revision_record_t *r_out, H5FD_t *raw_file, + const H5FD_onion_history_t *history, uint64_t revision_num); + +H5_DLL hbool_t H5FD__onion_archival_index_is_valid(const H5FD_onion_archival_index_t *); +H5_DLL int H5FD__onion_archival_index_find(const H5FD_onion_archival_index_t *, uint64_t, + const H5FD_onion_index_entry_t **); + +H5_DLL H5FD_onion_revision_index_t *H5FD__onion_revision_index_init(uint32_t page_size); +H5_DLL herr_t H5FD__onion_revision_index_destroy(H5FD_onion_revision_index_t *); +H5_DLL herr_t H5FD__onion_revision_index_insert(H5FD_onion_revision_index_t *, + const H5FD_onion_index_entry_t *); +H5_DLL int H5FD__onion_revision_index_find(const H5FD_onion_revision_index_t *, uint64_t, + const H5FD_onion_index_entry_t **); + +H5_DLL herr_t H5FD__onion_merge_revision_index_into_archival_index(const H5FD_onion_revision_index_t *, + H5FD_onion_archival_index_t *); + +H5_DLL size_t H5FD__onion_revision_record_decode(unsigned char *buf, H5FD_onion_revision_record_t *record); +H5_DLL size_t H5FD__onion_revision_record_encode(H5FD_onion_revision_record_t *record, unsigned char *buf, + uint32_t *checksum); + +#ifdef __cplusplus +} +#endif + +#endif /* H5FDonion_index_H */ diff --git a/src/H5FDonion_priv.h b/src/H5FDonion_priv.h new file mode 100644 index 0000000..031c132 --- /dev/null +++ b/src/H5FDonion_priv.h @@ -0,0 +1,28 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) Internals. + * + * Purpose: The private header file for the Onion VFD. + * Contains definitions and declarations used internallay and by + * tests. + */ + +#ifndef H5FDonion_priv_H +#define H5FDonion_priv_H + +#include "H5FDonion_header.h" +#include "H5FDonion_history.h" +#include "H5FDonion_index.h" + +#endif /* H5FDonion_priv_H */ diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index eb3d4cf..51527a0 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -46,6 +46,7 @@ #define H5_VFD_ROS3 ((H5FD_class_value_t)(11)) #define H5_VFD_SUBFILING ((H5FD_class_value_t)(12)) #define H5_VFD_IOC ((H5FD_class_value_t)(13)) +#define H5_VFD_ONION ((H5FD_class_value_t)(14)) /* VFD IDs below this value are reserved for library use. */ #define H5_VFD_RESERVED 256 diff --git a/src/H5private.h b/src/H5private.h index b3b8e97..f9e7aff 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -1068,6 +1068,9 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); #ifndef HDlog #define HDlog(X) log(X) #endif +#ifndef HDlog2 +#define HDlog2(X) log2(X) +#endif #ifndef HDlog10 #define HDlog10(X) log10(X) #endif diff --git a/src/H5trace.c b/src/H5trace.c index 8df2984..2b714fb 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -1098,6 +1098,9 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap) H5RS_acat(rs, "H5_VFD_ROS3"); break; #endif + case H5_VFD_ONION: + H5RS_acat(rs, "H5_VFD_ONION"); + break; default: H5RS_asprintf_cat(rs, "%ld", (long)class_val); break; diff --git a/src/Makefile.am b/src/Makefile.am index 0dbb175..a84fb02 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -61,8 +61,9 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \ H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ - H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c \ - H5FDmulti.c H5FDperform.c H5FDsec2.c H5FDspace.c \ + H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmulti.c \ + H5FDonion.c H5FDonion_header.c H5FDonion_history.c H5FDonion_index.c \ + H5FDperform.c H5FDsec2.c H5FDspace.c \ H5FDsplitter.c H5FDstdio.c H5FDtest.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ H5FSstat.c H5FStest.c \ @@ -154,9 +155,10 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers H5Cpublic.h H5Dpublic.h \ H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \ H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \ - H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \ - H5FDsec2.h H5FDsplitter.h \ - H5FDstdio.h H5FDwindows.h H5FDsubfiling/H5FDsubfiling.h H5FDsubfiling/H5FDioc.h \ + H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h \ + H5FDonion.h H5FDros3.h H5FDsec2.h H5FDsplitter.h \ + H5FDstdio.h H5FDsubfiling/H5FDsubfiling.h H5FDsubfiling/H5FDioc.h \ + H5FDwindows.h \ H5Gpublic.h H5Ipublic.h H5Lpublic.h \ H5Mpublic.h H5MMpublic.h H5Opublic.h H5Ppublic.h \ H5PLextern.h H5PLpublic.h \ @@ -64,6 +64,7 @@ #include "H5FDmirror.h" /* Mirror VFD and IPC definitions */ #include "H5FDmpi.h" /* MPI-based file drivers */ #include "H5FDmulti.h" /* Usage-partitioned file family */ +#include "H5FDonion.h" /* Onion file I/O */ #include "H5FDros3.h" /* R/O S3 "file" I/O */ #include "H5FDsec2.h" /* POSIX unbuffered file I/O */ #include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5bccc81..b9d1208 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -381,6 +381,7 @@ set (H5_TESTS timer cmpd_dtransform event_set + onion ) if (HDF5_BUILD_UTILS) set (H5_TESTS ${H5_TESTS} mirror_vfd) diff --git a/test/Makefile.am b/test/Makefile.am index fcfd34e..22510dc 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -70,7 +70,8 @@ TEST_PROG= testhdf5 \ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ - unregister cache_logging cork swmr thread_id vol timer event_set + unregister cache_logging cork swmr thread_id vol timer event_set \ + onion # List programs to be built when testing here # diff --git a/test/onion.c b/test/onion.c new file mode 100644 index 0000000..96a4970 --- /dev/null +++ b/test/onion.c @@ -0,0 +1,4966 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: + * + * Verify Onion VFD behavior that is not involved with operations on the + * backing store. + */ + +#include "h5test.h" + +#include "H5Fprivate.h" /* encode/decode macros */ +#include "H5FDonion.h" /* This file driver's utilities */ +#include "H5FDonion_priv.h" /* Onion file driver internals */ + +/* The Onion VFD uses H5MM calls internally, so any tests that allocate + * or free memory for said internal structures (e.g., the revision lists) + * will need to allocate memory using H5MM calls. + */ +#include "H5MMprivate.h" /* Memory management */ + +/* 2^n for uint64_t types -- H5_EXP2 unsafe past 32 bits */ +#define U64_EXP2(n) ((uint64_t)1 << (n)) + +#define ONION_TEST_PAGE_SIZE_1 4 +#define ONION_TEST_PAGE_SIZE_5 32 +#define ONION_TEST_BASENAME_SIZE 32 +#define ONION_TEST_FIXNAME_SIZE 1024 +#define ONION_TEST_EXPECTED_HISTORY_REVISIONS_MAX 16 +#define ONION_TEST_REV_REV_WRITES_MAX 8 +#define ONE_DIM_SIZE 1024 + +/* Structure to collect the onion filepaths in one place. */ +struct onion_filepaths { + char *canon; + char *onion; + char *recovery; +}; + +struct expected_revision { + uint64_t revision_num; + uint64_t parent_revision_num; + uint64_t logical_eof; + uint64_t n_index_entries; + const char *comment; +}; +struct expected_history { + uint64_t page_size; + uint64_t n_revisions; + uint64_t origin_eof; + struct expected_revision revisions[ONION_TEST_EXPECTED_HISTORY_REVISIONS_MAX]; +}; + +struct write_info { + haddr_t offset; + haddr_t size; + const unsigned char *buf; +}; +struct revise_revision { + hbool_t truncate; /* onion-create, truncating any existing data */ + uint64_t revision_num; + size_t n_writes; + struct write_info writes[ONION_TEST_REV_REV_WRITES_MAX]; + const char *comment; +}; + +static int compare_file_bytes_exactly(const char *filepath, hid_t fapl_id, size_t nbytes, + const unsigned char *exp); +static int do_onion_open_and_writes(const char *filename, H5FD_onion_fapl_info_t *onion_info_p, size_t n_ops, + struct revise_revision *about); +static void onion_filepaths_destroy(struct onion_filepaths *paths); +static struct onion_filepaths *onion_filepaths_init(const char *basename); + +/* set at runtime in main() */ +static unsigned int flags_create_s = 0; + +/* NOTE: b_list must be longer than a_list. + * Sizes must match respective buffer lengths. + */ + +/* twenty-six four-character words beginning with 'a' -> 104 bytes */ +const unsigned char *a_list_s = + (const unsigned char *)"abetableacedacesacheacidacneadzeafaragedagesaidsairsajarallyalum" + "amokantsapesarcsareaartsasksaspsavidaxes"; +uint64_t a_list_size_s = 104; + +/* fifty-three four-character words beginning with 'b' -> 212 bytes */ +const unsigned char *b_list_s = + (const unsigned char *)"badebailbaitbalebanebarebaskbeambeanbearbeenbeerbeltbentbestbide" + "bikebilebindbirdbiteblipblueboarboatbobsbodyboilboldbollboltbond" + "boneboobboorboosbootbradbragbratbraybrewbritbrowbuckbudsbunkbunt" + "buoyburnburybustbuys"; +uint64_t b_list_size_s = 212; + +/* Allocate and populate filepaths with h5_fixname'd strings as appropriate. + * Should be released with onion_filepaths_destroy() when done. + */ +static struct onion_filepaths * +onion_filepaths_init(const char *basename) +{ + struct onion_filepaths *paths = NULL; + + if (NULL == (paths = HDmalloc(sizeof(struct onion_filepaths)))) + TEST_ERROR; + paths->canon = NULL; + paths->onion = NULL; + paths->recovery = NULL; + + if (NULL == (paths->canon = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE))) + TEST_ERROR; + HDsnprintf(paths->canon, ONION_TEST_FIXNAME_SIZE, "%s", basename); + + if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE))) + TEST_ERROR; + HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon); + + if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE))) + TEST_ERROR; + HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon); + + return paths; + +error: + if (paths != NULL) { + HDfree(paths->canon); + HDfree(paths->onion); + HDfree(paths->recovery); + } + HDfree(paths); + + return NULL; +} + +/* Free onion file paths */ +static void +onion_filepaths_destroy(struct onion_filepaths *paths) +{ + HDfree(paths->canon); + HDfree(paths->onion); + HDfree(paths->recovery); + HDfree(paths); +} + +/*----------------------------------------------------------------------------- + * Function: test_archival_index() + * + * Purpose: Unit-test mechanisms for the onion archival index. + * Specifies and verifies index-validation and -search routines. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_archival_index(void) +{ + /* We can ignore each entry's physical address and checksum values */ + H5FD_onion_index_entry_t e0 = {1, 474}; + H5FD_onion_index_entry_t e1 = {4, 558}; + H5FD_onion_index_entry_t e2 = {5, 306}; + H5FD_onion_index_entry_t e3 = {9, 515}; + H5FD_onion_index_entry_t e4 = {14, 386}; + H5FD_onion_index_entry_t e5 = {18, 90}; + H5FD_onion_index_entry_t e6 = {19, 94}; + H5FD_onion_index_entry_t e7 = {20, 509}; + H5FD_onion_index_entry_t sorted[8] = {e0, e1, e2, e3, e4, e5, e6, e7}; + H5FD_onion_index_entry_t sorted_duplicates[8] = {e0, e1, e2, e2, e4, e5, e6, e7}; + H5FD_onion_index_entry_t sorted_incomplete[8] = {e1, e3, e4, e5}; + /* Partially-sorted list also aligned to 2 * page-size */ + H5FD_onion_index_entry_t sorted_partial[8] = {e1, e4, e5, e7, e0, e6, e2, e3}; /* 0..3 sorted */ + H5FD_onion_index_entry_t unsorted[8] = {e3, e1, e4, e5, e0, e6, e2, e7}; + H5FD_onion_archival_index_t aix = { + H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, 1, /* page_size_log2 */ + 8, /* list must be populated and sorted through 0 .. (count-1) */ + sorted, /* list */ + }; + const H5FD_onion_index_entry_t *entry_out_p = NULL; + + TESTING("archival index"); + + /* + * Failing validity checks + */ + + /* Invalid version should fail */ + aix.version++; + if (H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + /* Invalid version should fail */ + aix.version = 0; + if (H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + aix.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR; + + /* NULL list should fail */ + aix.list = NULL; + if (H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + /* List not full should fail */ + aix.list = sorted_incomplete; + if (H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + /* Unsorted list should fail */ + aix.list = unsorted; + if (H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + /* List with duplicates should fail */ + aix.list = sorted_duplicates; + if (H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + /* + * Passing validity checks + */ + + /* Sorted list should pass */ + aix.list = sorted; + if (!H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + /* Extra elements ignored (should pass) */ + aix.list = sorted_partial; + aix.n_entries = 4; + if (!H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + /* + * Archival index search routine + */ + + aix.list = sorted; + aix.n_entries = 8; + + /* Check that address not in array returns zero */ + if (H5FD__onion_archival_index_find(&aix, 3, &entry_out_p) != 0) + TEST_ERROR; + /* Pointer should remain unset */ + if (entry_out_p != NULL) + TEST_ERROR; + + /* Address found should return 1 */ + if (H5FD__onion_archival_index_find(&aix, 4, &entry_out_p) != 1) + TEST_ERROR; + /* Pointer should be set */ + if (NULL == entry_out_p) + TEST_ERROR; + /* Incorrect address recorded */ + if (558 != entry_out_p->phys_addr) + TEST_ERROR; + + /* + * Test search edge cases + */ + + aix.list = sorted_incomplete; + aix.n_entries = 4; + + /* Address not in array should return 0 */ + if (H5FD__onion_archival_index_find(&aix, 1, &entry_out_p) != 0) + TEST_ERROR; + + /* Address not in array should return 0 */ + if (H5FD__onion_archival_index_find(&aix, 101, &entry_out_p) != 0) + TEST_ERROR; + + /* + * Empty archival index + */ + + entry_out_p = NULL; + aix.n_entries = 0; /* actually populated list is irrelevant */ + /* Address not in array should return 0 */ + if (H5FD__onion_archival_index_find(&aix, 3, &entry_out_p) != 0) + TEST_ERROR; + /* Pointer should remain unset */ + if (entry_out_p != NULL) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return -1; +} /* end test_archival_index() */ + +/*----------------------------------------------------------------------------- + * Function: test_revision_index() + * + * Purpose: Test revision index functionality + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_revision_index(void) +{ + H5FD_onion_revision_index_t *rix_p = NULL; + H5FD_onion_index_entry_t entry = { + 42, /* logical_page */ + 111112, /* phys_addr */ + }; + const H5FD_onion_index_entry_t *entry_out_p = NULL; + + TESTING("revision index"); + + /* Test index creation */ + + if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5))) + TEST_ERROR; + if (H5FD_ONION_REVISION_INDEX_VERSION_CURR != rix_p->version) + TEST_ERROR; + if (0 != rix_p->n_entries) + TEST_ERROR; + + /* Test missed search */ + + if (H5FD__onion_revision_index_find(rix_p, entry.logical_page, &entry_out_p) != 0) + TEST_ERROR; + + /* Test successful insertion and lookup */ + + /* Insertion failed */ + if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + if (1 != rix_p->n_entries) + TEST_ERROR; + /* Lookup failed */ + if (H5FD__onion_revision_index_find(rix_p, entry.logical_page, &entry_out_p) < 0) + TEST_ERROR; + /* Failure to set output parameter */ + if (NULL == entry_out_p) + TEST_ERROR; + if (entry.logical_page != entry_out_p->logical_page) + TEST_ERROR; + /* Seeking missing page should miss */ + if (H5FD__onion_revision_index_find(rix_p, entry.logical_page + 1, &entry_out_p) != 0) + TEST_ERROR; + + /* Test / demonstrate stored entry independent of user object */ + + entry.logical_page = 100; + entry.phys_addr = 101; + if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + if (2 != rix_p->n_entries) + TEST_ERROR; + entry.logical_page = 500; + entry.phys_addr = 501; + if (H5FD__onion_revision_index_find(rix_p, 100, &entry_out_p) < 0) + TEST_ERROR; + if (100 != entry_out_p->logical_page || 101 != entry_out_p->phys_addr) + TEST_ERROR; + + /* Demonstrate updating an entry */ + + /* Error cases */ + + entry.logical_page = 100; /* phys_addr still 501, checksum bbbbbbbb */ + if (H5FD__onion_revision_index_insert(rix_p, &entry) >= 0) + TEST_ERROR; /* all components but sum must match */ + entry.phys_addr = 101; + + /* Successful update */ + + entry.logical_page = 100; + entry.phys_addr = 101; + if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + + /* Should still be two unique entries, not three */ + if (2 != rix_p->n_entries) + TEST_ERROR; + if (H5FD__onion_revision_index_find(rix_p, 100, &entry_out_p) < 0) + TEST_ERROR; + if (100 != entry_out_p->logical_page || 101 != entry_out_p->phys_addr) + TEST_ERROR; + + if (H5FD__onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + + PASSED(); + return 0; + +error: + if (rix_p != NULL) + (void)H5FD__onion_revision_index_destroy(rix_p); + + return -1; +} /* end test_revision_index() */ + +/*----------------------------------------------------------------------------- + * Function: test_revision_index_collisions() + * + * Purpose: With knowledge of the revision index implementation, test + * hash key collisions. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_revision_index_collisions(void) +{ + H5FD_onion_revision_index_t *rix_p = NULL; + H5FD_onion_index_entry_t entry = { + 0, /* logical_page */ + 0, /* phys_addr */ + }; + const H5FD_onion_index_entry_t *entry_out_p = NULL; + const uint64_t n_insert = 40; + const uint64_t offset_from_power = 5; + + TESTING("revision index collisions"); + + if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5))) + TEST_ERROR; + + for (uint64_t i = 0; i < n_insert; i++) { + entry.phys_addr = i; + entry.logical_page = U64_EXP2(i) + offset_from_power; + if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + } + + if (n_insert != rix_p->n_entries) + TEST_ERROR; + + for (uint64_t i = 0; i < n_insert; i++) { + uint64_t page_id = U64_EXP2(i) + offset_from_power; + + if (H5FD__onion_revision_index_find(rix_p, page_id, &entry_out_p) != 1) + TEST_ERROR; + if (entry_out_p->phys_addr != i) + TEST_ERROR; + } + + if (H5FD__onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + + PASSED(); + return 0; + +error: + if (rix_p != NULL) + (void)H5FD__onion_revision_index_destroy(rix_p); + + return -1; +} /* end test_revision_index_collisions() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_revision_index_resizing() + * + * Purpose: With knowledge of the revision index implementation, test + * one or more resizig of the index. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_revision_index_resizing(void) +{ + H5FD_onion_revision_index_t *rix_p = NULL; + H5FD_onion_index_entry_t entry = { + 0, /* logical_page */ + 0, /* phys_addr */ + }; + const H5FD_onion_index_entry_t *entry_out_p = NULL; + const uint64_t n_insert = U64_EXP2((H5FD_ONION_REVISION_INDEX_STARTING_SIZE_LOG2 + 3)); + + TESTING("revision index resizing"); + + if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5))) + TEST_ERROR; + + for (uint64_t i = 0; i < n_insert; i++) { + entry.logical_page = i; + entry.phys_addr = ((uint64_t)(-1) - i); + if (H5FD__onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + } + + if (n_insert != rix_p->n_entries) + TEST_ERROR; + + for (uint64_t i = 0; i < n_insert; i++) { + uint64_t page_id = i; + + if (H5FD__onion_revision_index_find(rix_p, page_id, &entry_out_p) != 1) + TEST_ERROR; + if (entry_out_p->phys_addr != ((uint64_t)(-1) - i)) + TEST_ERROR; + } + + if (H5FD__onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + + PASSED(); + return 0; + +error: + if (rix_p != NULL) + (void)H5FD__onion_revision_index_destroy(rix_p); + + return -1; +} /* end test_revision_index_resizing() */ + +/*----------------------------------------------------------------------------- + * Function: test_revision_index_to_archival_index() + * + * Purpose: Verify to_archival_index(). + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_revision_index_to_archival_index(void) +{ + H5FD_onion_revision_index_t *rix_p = NULL; + H5FD_onion_index_entry_t rix_entry = { + 0, /* logical_page */ + 0, /* phys_addr */ + }; + H5FD_onion_archival_index_t aix = { + H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, + 5, /* page_size_log2 */ + 0, /* n_entries to be set */ + NULL, + }; + const uint64_t n_insert = 10; + + TESTING("revision index to archival index"); + + /* + * SETUP + */ + + if (NULL == (rix_p = H5FD__onion_revision_index_init(ONION_TEST_PAGE_SIZE_5))) + TEST_ERROR; + + /* Add scattered entries in reverse order. */ + for (uint64_t i = 0; i < n_insert; i++) { + uint64_t n = 2003 * (n_insert - i) + 47; + + rix_entry.logical_page = n; + rix_entry.phys_addr = n * 13; + if (H5FD__onion_revision_index_insert(rix_p, &rix_entry) < 0) + TEST_ERROR; + } + + if (n_insert != rix_p->n_entries) + TEST_ERROR; + + aix.list = NULL; + aix.n_entries = 0; + + /* Successful merge into empty archival index */ + + if (H5FD__onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0) + TEST_ERROR; + + if (!H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + if (n_insert != aix.n_entries) + TEST_ERROR; + for (uint64_t i = 0; i < n_insert; i++) { + const H5FD_onion_index_entry_t *aix_entry_p = NULL; + + uint64_t n = 2003 * (i + 1) + 47; + + aix_entry_p = &aix.list[i]; + + if (aix_entry_p->logical_page != n) + TEST_ERROR; + if (aix_entry_p->phys_addr != (n * 13)) + TEST_ERROR; + } + + /* Successful merge into populated archival index */ + + H5MM_xfree(aix.list); + aix.list = NULL; + if (NULL == (aix.list = H5MM_malloc(sizeof(H5FD_onion_index_entry_t) * 2))) + TEST_ERROR; + aix.list[0].logical_page = 47; + aix.list[0].phys_addr = 47 * 13; + aix.list[1].logical_page = (2003 * (n_insert + 1) + 47); + aix.list[1].phys_addr = (2003 * (n_insert + 1) + 47) * 13; + aix.n_entries = 2; + + if (!H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + if (H5FD__onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0) + TEST_ERROR; + if (!H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + if (n_insert + 2 != aix.n_entries) + TEST_ERROR; + + for (uint64_t i = 0; i < (n_insert + 2); i++) { + const H5FD_onion_index_entry_t *aix_entry_p = NULL; + + uint64_t n = 2003 * i + 47; + + aix_entry_p = &aix.list[i]; + + if (aix_entry_p->logical_page != n) + TEST_ERROR; + if (aix_entry_p->phys_addr != (n * 13)) + TEST_ERROR; + } + + /* Merged enties from revision index replace existing entries */ + + H5MM_xfree(aix.list); + aix.list = NULL; + if (NULL == (aix.list = H5MM_malloc(sizeof(H5FD_onion_index_entry_t) * 2))) + TEST_ERROR; + aix.list[0].logical_page = 2003 * (n_insert / 2) + 47; + aix.list[0].phys_addr = 103; + aix.list[1].logical_page = 2003 * (n_insert / 2 + 1) + 47; + aix.list[1].phys_addr = 101; + aix.n_entries = 2; + + if (!H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + if (H5FD__onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0) + TEST_ERROR; + + if (!H5FD__onion_archival_index_is_valid(&aix)) + TEST_ERROR; + + if (n_insert != aix.n_entries) + TEST_ERROR; + + for (uint64_t i = 0; i < n_insert; i++) { + const H5FD_onion_index_entry_t *aix_entry_p = NULL; + uint64_t n = 2003 * (i + 1) + 47; + + aix_entry_p = &aix.list[i]; + + if (aix_entry_p->logical_page != n) + TEST_ERROR; + if (aix_entry_p->phys_addr != (n * 13)) + TEST_ERROR; + } + + /* CLEANUP */ + + if (H5FD__onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + H5MM_xfree(aix.list); + + PASSED(); + return 0; + +error: + if (rix_p) + (void)H5FD__onion_revision_index_destroy(rix_p); + H5MM_xfree(aix.list); + + return -1; +} /* end test_revision_index_to_archival_index() */ + +/*----------------------------------------------------------------------------- + * Function: test_fapl() + * + * Purpose: Verify H5Pget and set behavior, and data-consistency checks. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_fapl(void) +{ + H5FD_onion_fapl_info_t info_in = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_1, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "indoor speaking voices", /* comment */ + }; + H5FD_onion_fapl_info_t info_out; + hid_t dxpl_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t fapl_id_sec2 = H5I_INVALID_HID; + herr_t ret = FAIL; + + TESTING("file access property list"); + + if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + if ((fapl_id_sec2 = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_sec2(fapl_id_sec2)) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + /* Set FAPL */ + + /* Invalid fapl should fail */ + H5E_BEGIN_TRY + { + ret = H5Pset_fapl_onion(H5I_INVALID_HID, &info_in); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + + /* NULL info pointer should fail */ + H5E_BEGIN_TRY + { + ret = H5Pset_fapl_onion(fapl_id, NULL); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + + /* Invalid version should fail */ + info_in.version++; + H5E_BEGIN_TRY + { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + info_in.version--; + + /* Page size not a power of 2 should fail */ + info_in.page_size = 7; + H5E_BEGIN_TRY + { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + + /* Page size <=0 should fail */ + info_in.page_size = 0; + H5E_BEGIN_TRY + { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + info_in.page_size = ONION_TEST_PAGE_SIZE_1; + + /* Invalid backing fapl should fail */ + info_in.backing_fapl_id = H5I_INVALID_HID; + H5E_BEGIN_TRY + { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + + /* Backing fapl not a fapl should fail */ + info_in.backing_fapl_id = dxpl_id; + H5E_BEGIN_TRY + { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + info_in.backing_fapl_id = H5P_DEFAULT; + + if (H5Pset_fapl_onion(fapl_id, &info_in) < 0) + TEST_ERROR; + + /* Get onion fapl info */ + + /* NULL info_out pointer should fail */ + H5E_BEGIN_TRY + { + ret = H5Pget_fapl_onion(fapl_id, NULL); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + + /* Invalid fapl should fail */ + H5E_BEGIN_TRY + { + ret = H5Pget_fapl_onion(H5I_INVALID_HID, &info_out); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + + /* Non-onion fapl ID should fail */ + H5E_BEGIN_TRY + { + ret = H5Pget_fapl_onion(fapl_id_sec2, &info_out); + } + H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; + + /* Normal case */ + if (H5Pget_fapl_onion(fapl_id, &info_out) < 0) + TEST_ERROR; + if (H5FD_ONION_FAPL_INFO_VERSION_CURR != info_out.version) + TEST_ERROR; + if (H5P_DEFAULT != info_out.backing_fapl_id) + TEST_ERROR; + if (ONION_TEST_PAGE_SIZE_1 != info_out.page_size) + TEST_ERROR; + if (H5FD_ONION_STORE_TARGET_ONION != info_out.store_target) + TEST_ERROR; + if (H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST != info_out.revision_num) + TEST_ERROR; + if (0 != info_out.creation_flags) + TEST_ERROR; + if (0 != info_out.force_write_open) + TEST_ERROR; + if (HDstrcmp(info_in.comment, info_out.comment)) + TEST_ERROR; + + /* Cleanup */ + + if (H5Pclose(dxpl_id) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id_sec2) < 0) + TEST_ERROR; + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(dxpl_id); + H5Pclose(fapl_id); + H5Pclose(fapl_id_sec2); + } + H5E_END_TRY + + return -1; +} /* end test_fapl() */ + +/*----------------------------------------------------------------------------- + * Function: test_header_encode_decode() + * + * Purpose: Verify onion header encoding and decoding behavior. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_header_encode_decode(void) +{ + unsigned char buf[64]; + unsigned char exp[64] = { + /* bogus but unique values */ + 'O', 'H', 'D', 'H', /* NOTE: match signature define in onion_priv.h */ + 1, 12, 0, 0, /* NOTE: update version w/ "current" as needed */ + 0, 16, 0, 0, 0x11, 0x00, 0, 0, 0x02, 0, 0, 0, /* origin_eof */ + 0x40, 0xe2, 0x01, 0, 0, 0, 0, 0, /* history_addr */ + 88, 0, 0, 0, 0, 0, 0, 0, /* history_size */ + 0, 0, 0, 0 /* sum populated below */ + }; + unsigned char *ptr = NULL; + uint32_t checksum = 0; + uint32_t checksum_out = 0; + size_t i = 0; + uint64_t size_ret = 0; + H5FD_onion_header_t hdr; + H5FD_onion_header_t hdr_out; + + TESTING("encode/decode history header"); + + checksum = H5_checksum_fletcher32(exp, H5FD_ONION_ENCODED_SIZE_HEADER - 4); + ptr = exp + H5FD_ONION_ENCODED_SIZE_HEADER - 4; + UINT32ENCODE(ptr, checksum); + + hdr.version = H5FD_ONION_HEADER_VERSION_CURR; + hdr.flags = 12; + hdr.origin_eof = 8589934609ull, hdr.page_size = 4096; + hdr.history_addr = 123456; + hdr.history_size = 88; + + if (H5FD__onion_header_encode(&hdr, buf, &checksum_out) != H5FD_ONION_ENCODED_SIZE_HEADER) + TEST_ERROR; + + if (checksum != checksum_out) + TEST_ERROR; + + for (i = 0; i < H5FD_ONION_ENCODED_SIZE_HEADER; i++) { + if (exp[i] != buf[i]) { + HDprintf("first mismatched byte at %zu: %02x %02x\n", i, exp[i], buf[i]); + TEST_ERROR; + } + } + + hdr_out.version = H5FD_ONION_HEADER_VERSION_CURR; + hdr_out.flags = 0; + hdr_out.page_size = 0; + hdr_out.history_addr = 0; + hdr_out.history_size = 0; + + /* Invalid header signature prevents decoding. + */ + + exp[3] = 'X'; /* invalidate encoded signature */ + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_header_decode(exp, &hdr_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[3] = 'H'; /* reset */ + + /* Invalid header version prevents decoding. + */ + + exp[4] = 0; /* encoded version 0?!? */ + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_header_decode(exp, &hdr_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[4] = H5FD_ONION_HEADER_VERSION_CURR + 1; /* encoded super-version?! */ + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_header_decode(exp, &hdr_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + /* Reset */ + exp[4] = H5FD_ONION_HEADER_VERSION_CURR; + + /* Valid header can be decoded */ + + if (H5FD__onion_header_decode(buf, &hdr_out) != H5FD_ONION_ENCODED_SIZE_HEADER) + TEST_ERROR; + if (H5FD_ONION_HEADER_VERSION_CURR != hdr_out.version) + TEST_ERROR; + if (hdr.flags != hdr_out.flags) + TEST_ERROR; + if (hdr.page_size != hdr_out.page_size) + TEST_ERROR; + if (hdr.history_addr != hdr_out.history_addr) + TEST_ERROR; + if (hdr.history_size != hdr_out.history_size) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return -1; +} /* end test_header_encode_decode() */ + +/*----------------------------------------------------------------------------- + * Function: test_history_encode_decode_empty() + * + * Purpose: Verify onion history encoding and decoding behavior. + * Tests the case of the "empty" history. + * Verifies behavior in standard error cases. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_history_encode_decode_empty(void) +{ + unsigned char buf[32]; + unsigned char exp[32] = { + 'O', 'W', 'H', 'S', /* NOTE: match signature define in onion_priv.h */ + 1, 0, 0, 0, /* NOTE: update version w/ "current" as needed */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* sum populated below */ + }; + unsigned char *ptr = NULL; + uint32_t checksum = 0; + uint32_t checksum_out = 0; + size_t i = 0; + uint64_t size_ret = 0; + H5FD_onion_history_t history = { + H5FD_ONION_HISTORY_VERSION_CURR, 0, /* n_revisions */ + NULL, /* list */ + 0, /* checksum */ + }; + H5FD_onion_history_t history_out = { + H5FD_ONION_HISTORY_VERSION_CURR, 0, /* n_revisions */ + NULL, /* list */ + 0, /* checksum */ + }; + + TESTING("encode/decode history (empty and failures)"); + + /* Generage checksum but don't store it yet */ + checksum = H5_checksum_fletcher32(exp, H5FD_ONION_ENCODED_SIZE_HISTORY - 4); + ptr = exp + H5FD_ONION_ENCODED_SIZE_HISTORY - 4; + UINT32ENCODE(ptr, checksum); + + if (H5FD__onion_history_encode(&history, buf, &checksum_out) != H5FD_ONION_ENCODED_SIZE_HISTORY) + TEST_ERROR; + for (i = 0; i < 20; i++) { + if (exp[i] != buf[i]) { + HDprintf("first mismatched byte at %zu: %02x %02x\n", i, exp[i], buf[i]); + TEST_ERROR; + } + } + if (checksum != checksum_out) + TEST_ERROR; + history.checksum = checksum; /* set to compare later */ + + /* Invalid signature prevents decoding */ + + exp[3] = 'X'; /* invalidate encoded signature */ + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_history_decode(exp, &history_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[3] = 'H'; /* reset */ + + /* Invalid version prevents decoding */ + + exp[4] = 0; /* encoded version 0?!? */ + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_history_decode(exp, &history_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[4] = H5FD_ONION_HISTORY_VERSION_CURR + 1; + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_history_decode(exp, &history_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[4] = H5FD_ONION_HISTORY_VERSION_CURR; /* reset */ + + /* Valid summary can be decoded */ + + if (H5FD__onion_history_decode(buf, &history_out) != H5FD_ONION_ENCODED_SIZE_HISTORY) + TEST_ERROR; + if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version) + TEST_ERROR; + if (history.n_revisions != history_out.n_revisions) + TEST_ERROR; + if (history.checksum != history_out.checksum) + TEST_ERROR; + if (NULL != history_out.record_locs) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return -1; +} /* end test_history_encode_decode_empty() */ + +/*----------------------------------------------------------------------------- + * Function: test_history_encode_decode() + * + * Purpose: Verify onion history encoding and decoding behavior. + * Encode/decode with some set of revision record pointers. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_history_encode_decode(void) +{ + unsigned char *buf = NULL; + unsigned char exp[80] = { + 'O', 'W', 'H', 'S', /* NOTE: match signature define in onion_priv.h */ + 1, 0, 0, 0, /* NOTE: update version w/ "current" as needed */ + 3, 0, 0, 0, 0, 0, 0, 0, + /* rev0 pointer */ + 56, 2, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* sum populated below */ + /* rev1 pointer */ + 121, 173, 3, 0, 0, 0, 0, 0, 203, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* sum populated below */ + /* rev2 pointer */ + 96, 158, 52, 198, 213, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* sum populated below */ + /* final checksum */ + 0, 0, 0, 0 /* sum populated below */ + }; + unsigned char *buf_p = NULL; + uint32_t checksum_out = 0; + size_t i = 0; + H5FD_onion_history_t history = { + H5FD_ONION_HISTORY_VERSION_CURR, 3, /* n_revisions */ + NULL, /* list set below */ + 0, /* checksum not set by us */ + }; + H5FD_onion_history_t history_out = { + H5FD_ONION_HISTORY_VERSION_CURR, 0, /* n_revisions must start as zero */ + NULL, /* list */ + 0, /* checksum */ + }; + size_t exp_size = + H5FD_ONION_ENCODED_SIZE_HISTORY + H5FD_ONION_ENCODED_SIZE_RECORD_POINTER * history.n_revisions; + + TESTING("encode/decode history"); + + if (80 != exp_size) + TEST_ERROR; + + history.record_locs = HDcalloc(history.n_revisions, sizeof(H5FD_onion_record_loc_t)); + if (NULL == history.record_locs) + TEST_ERROR; + + /* Must match values in exp */ + history.record_locs[0].phys_addr = 568ull; + history.record_locs[0].record_size = 238ull; + history.record_locs[1].phys_addr = 241017ull; + history.record_locs[1].record_size = 4555ull; + history.record_locs[2].phys_addr = 918153371232ull; + history.record_locs[2].record_size = 240ull; + + /* Populate revision pointer sums in exp */ + for (i = 0; i < history.n_revisions; i++) { + uint64_t history_pre = H5FD_ONION_ENCODED_SIZE_HISTORY - 4; + uint64_t ptr_pre = H5FD_ONION_ENCODED_SIZE_RECORD_POINTER - 4; + uint64_t ptr_size = H5FD_ONION_ENCODED_SIZE_RECORD_POINTER; + + buf_p = exp + history_pre + ptr_size * i; + history.record_locs[i].checksum = H5_checksum_fletcher32(buf_p, ptr_pre); + buf_p += ptr_pre; + UINT32ENCODE(buf_p, history.record_locs[i].checksum); + } + + /* Compute, populate, and store exp final sum */ + history.checksum = H5_checksum_fletcher32(exp, exp_size - 4); + buf_p = exp + exp_size - 4; + UINT32ENCODE(buf_p, history.checksum); + + if (NULL == (buf = HDmalloc(exp_size))) + TEST_ERROR; + + if (H5FD__onion_history_encode(&history, buf, &checksum_out) != exp_size) + TEST_ERROR; + for (i = 0; i < exp_size; i++) { + if (exp[i] != buf[i]) + TEST_ERROR; + } + if (history.checksum != checksum_out) + TEST_ERROR; + + /* Initial decode, gets always-present components */ + + history_out.n_revisions = 0; /* must be initialized to 0 */ + if (H5FD__onion_history_decode(exp, &history_out) != exp_size) + TEST_ERROR; + if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version) + TEST_ERROR; + if (history.n_revisions != history_out.n_revisions) + TEST_ERROR; + /* Must be created by us */ + if (NULL != history_out.record_locs) + TEST_ERROR; + + /* True decode requires allocating space for record pointers */ + + history_out.record_locs = HDcalloc(history_out.n_revisions, sizeof(H5FD_onion_record_loc_t)); + if (NULL == history_out.record_locs) + TEST_ERROR; + + if (H5FD__onion_history_decode(exp, &history_out) != exp_size) + TEST_ERROR; + if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version) + TEST_ERROR; + if (history.n_revisions != history_out.n_revisions) + TEST_ERROR; + if (history.checksum != history_out.checksum) + TEST_ERROR; + if (NULL == history_out.record_locs) + TEST_ERROR; + for (i = 0; i < history.n_revisions; i++) { + H5FD_onion_record_loc_t exp_rp = history.record_locs[i]; + H5FD_onion_record_loc_t act_rp = history_out.record_locs[i]; + + if (exp_rp.phys_addr != act_rp.phys_addr) + TEST_ERROR; + if (exp_rp.record_size != act_rp.record_size) + TEST_ERROR; + if (exp_rp.checksum != act_rp.checksum) + TEST_ERROR; + } + + HDfree(history_out.record_locs); + HDfree(buf); + HDfree(history.record_locs); + + PASSED(); + return 0; + +error: + HDfree(history_out.record_locs); + HDfree(buf); + HDfree(history.record_locs); + + return -1; +} /* end test_history_encode_decode() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_revision_record_encode_decode() + * + * Purpose: Verify onion revision-record encoding and decoding behavior. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_revision_record_encode_decode(void) +{ + /* clang-format off */ + /* Byte array of expected values (FRAGILE!) */ + unsigned char exp[173] = { + 'O', 'R', 'R', 'S', /* Bytes 000-003: signature */ + 1, 0, 0, 0, /* Bytes 004-007: version */ + 5, 0, 0, 0, 0, 0, 0, 0, /* Bytes 008-015: revision ID */ + 2, 0, 0, 0, 0, 0, 0, 0, /* Bytes 016-023: parent revision ID */ + '1', '9', '4', '1', '1', '2', '0', '7', /* Bytes 024-039: time of creation */ + 'T', '1', '9', '0', '6', '4', '3', 'Z', /* ... */ + 0x11, 0x00, 0, 0, 0x02, 0, 0, 0, /* Bytes 040-047: logical file size */ + 0, 16, 0, 0, /* Bytes 048-051: page size */ + 4, 0, 0, 0, 0, 0, 0, 0, /* Bytes 052-059: # entries */ + 25, 0, 0, 0, /* Bytes 060-063: comment size */ + /* ENTRY 0 */ + 0, 0xB0, 0x1E, 0, 0, 0, 0, 0, /* Bytes 064-071: entry 0: logical offset */ + 0x4B, 0x02, 0, 0, 0, 0, 0, 0, /* Bytes 072-079: entry 0: physical address */ + 0, 0, 0, 0, /* Bytes 080-083: checksum (populated below) */ + /* ENTRY 1 */ + 0, 0xF0, 0x2E, 0, 0, 0, 0, 0, /* Bytes 084-091: entry 1: logical offset */ + 0xA7, 0, 0, 0, 0, 0, 0, 0, /* Bytes 092-099: entry 1: physical address */ + 0, 0, 0, 0, /* Bytes 100-103: checksum (populated below) */ + /* ENTRY 2 */ + 0, 0x50, 0x15, 0, 0, 0x20, 0, 0, /* Bytes 104-111: entry 2: logical offset */ + 0x11, 0, 0, 0, 0x02, 0, 0, 0, /* Bytes 112-119: entry 2: physical address */ + 0, 0, 0, 0, /* Bytes 120-123: checksum (populated below) */ + /* ENTRY 3 */ + 0, 0xE0, 0x24, 0, 0, 0, 0, 0, /* Bytes 124-131: entry 3: logical offset */ + 0xB1, 0x01, 0, 0, 0, 0, 0, 0, /* Bytes 132-139: entry 3: physical address */ + 0, 0, 0, 0, /* Bytes 140-143: checksum (populated below) */ + 'E', 'x', 'a', 'm', 'p', 'l', 'e', ' ', /* Bytes 144-168: comment */ + 'c', 'o', 'm', 'm', 'e', 'n', 't', ' ', /* ... */ + 'm', 'e', 's', 's', 'a', 'g', 'e', '.', /* ... */ + '\0', /* ... */ + 0, 0, 0, 0 /* Bytes 169-172: final checksum (populated below) */ + }; + /* clang-format on */ + unsigned char *buf = NULL; + unsigned char *buf_p = NULL; + size_t i = 0; + uint64_t size_ret; + H5FD_onion_revision_record_t r_out; + uint32_t checksum = 0; + char comment[25] = "Example comment message."; + H5FD_onion_revision_record_t record = { + H5FD_ONION_REVISION_RECORD_VERSION_CURR, + 5, /* revision ID */ + 2, /* parent revision ID */ + {'\0'}, /* time of creation - populated below */ + 8589934609ull, /* logical file size */ + { + H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR, /* version */ + 12, /* page_size_log2 */ + 4, /* n_entries */ + NULL, /* list - populated below */ + }, /* archival index struct */ + 25, /* comment size */ + comment, /* comment */ + 0, /* checksum (computed later) */ + }; + size_t exp_size = H5FD_ONION_ENCODED_SIZE_REVISION_RECORD + + (H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY * record.archival_index.n_entries) + + HDstrlen("Example comment message.") + 1; + + r_out.archival_index.list = NULL; + r_out.comment = NULL; + + TESTING("encode/decode revision record"); + + HDmemcpy(record.time_of_creation, "19411207T190643Z", 16); + record.archival_index.list = HDcalloc(record.archival_index.n_entries, sizeof(H5FD_onion_index_entry_t)); + if (NULL == record.archival_index.list) + TEST_ERROR; + + /* Convert logical_page and should match address in expected buffer */ + record.archival_index.list[0].logical_page = 491ull; + record.archival_index.list[0].phys_addr = 587ull; + record.archival_index.list[1].logical_page = 751ull; + record.archival_index.list[1].phys_addr = 167ull; + record.archival_index.list[2].logical_page = 8589934933ull; + record.archival_index.list[2].phys_addr = 8589934609ull; + record.archival_index.list[3].logical_page = 590ull; + record.archival_index.list[3].phys_addr = 433ull; + + /* Set expected checksum for each archival index entry in buffer */ + for (i = 0; i < record.archival_index.n_entries; i++) { + uint64_t rec_pre = H5FD_ONION_ENCODED_SIZE_REVISION_RECORD - 4; + uint64_t idx_pre = H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY - 4; + uint64_t idx_size = H5FD_ONION_ENCODED_SIZE_INDEX_ENTRY; + + buf_p = exp + rec_pre + idx_size * i; + checksum = H5_checksum_fletcher32(buf_p, idx_pre); + buf_p += idx_pre; + UINT32ENCODE(buf_p, checksum); + } + + checksum = 0; + + record.checksum = H5_checksum_fletcher32(exp, exp_size - 4); + buf_p = exp + exp_size - 4; + UINT32ENCODE(buf_p, record.checksum); + + /* Required initialization for record-out structure */ + r_out.version = H5FD_ONION_REVISION_RECORD_VERSION_CURR; + r_out.comment_size = 0; + r_out.comment = NULL; + r_out.archival_index.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR; + r_out.archival_index.n_entries = 0; + r_out.archival_index.list = NULL; + + if (NULL == (buf = HDmalloc(sizeof(unsigned char) * exp_size))) + TEST_ERROR; + + /* Test encode */ + + if (H5FD__onion_revision_record_encode(&record, buf, &checksum) != exp_size) + TEST_ERROR; + + hbool_t badness = FALSE; + for (i = 0; i < exp_size; i++) { + if (exp[i] != buf[i]) { + badness = TRUE; + HDprintf("Bad encoded record - Index %zu: expected 0x%02X but got 0x%02X\n", i, + (unsigned int)exp[i], (unsigned int)buf[i]); + } + } + if (badness) { + /* If this fragile test breaks, this information is helpful... */ + HDprintf("INDEX\n"); + for (i = 0; i < exp_size; i++) + HDprintf("%4zu ", i); + HDprintf("\n"); + + HDprintf("EXPECTED\n"); + for (i = 0; i < exp_size; i++) + HDprintf("0x%02X ", (unsigned int)exp[i]); + HDprintf("\n"); + + HDprintf("ACTUAL\n"); + for (i = 0; i < exp_size; i++) + HDprintf("0x%02X ", (unsigned int)buf[i]); + HDprintf("\n"); + } + if (badness) + TEST_ERROR; + if (record.checksum != checksum) + TEST_ERROR; + + /* Test decode (malformed encoding) */ + + /* Invalid signature */ + exp[2] = 'Y'; + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_revision_record_decode(exp, &r_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + exp[2] = 'R'; /* reset */ + + /* Zero version */ + exp[4] = 0; + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_revision_record_decode(exp, &r_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + /* Advance version */ + exp[4] = H5FD_ONION_REVISION_RECORD_VERSION_CURR + 1; + H5E_BEGIN_TRY + { + size_ret = H5FD__onion_revision_record_decode(exp, &r_out); + } + H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + exp[4] = H5FD_ONION_REVISION_RECORD_VERSION_CURR; /* reset */ + + /* Test successful decode */ + + /* Initial decode; get variable-length component sizes */ + if (H5FD__onion_revision_record_decode(exp, &r_out) != exp_size) + TEST_ERROR; + if (record.comment_size != r_out.comment_size) + TEST_ERROR; + if (record.archival_index.n_entries != r_out.archival_index.n_entries) + TEST_ERROR; + + /* Allocate variable-length components */ + r_out.comment = HDcalloc(r_out.comment_size, sizeof(char)); + if (NULL == r_out.comment) + TEST_ERROR; + r_out.archival_index.list = HDcalloc(r_out.archival_index.n_entries, sizeof(H5FD_onion_index_entry_t)); + if (NULL == r_out.archival_index.list) + TEST_ERROR; + + /* Decode into all components */ + if (H5FD__onion_revision_record_decode(exp, &r_out) != exp_size) + TEST_ERROR; + if (H5FD_ONION_REVISION_RECORD_VERSION_CURR != r_out.version) + TEST_ERROR; + if (record.revision_num != r_out.revision_num) + TEST_ERROR; + if (record.parent_revision_num != r_out.parent_revision_num) + TEST_ERROR; + if (record.parent_revision_num != r_out.parent_revision_num) + TEST_ERROR; + if (record.checksum != r_out.checksum) + TEST_ERROR; + if (HDstrncmp(record.time_of_creation, r_out.time_of_creation, 16) != 0) + TEST_ERROR; + if (record.comment_size != r_out.comment_size) + TEST_ERROR; + if (record.comment_size != HDstrlen(r_out.comment) + 1) + TEST_ERROR; + if (HDstrlen(record.comment) != HDstrlen(r_out.comment)) + TEST_ERROR; + if (HDstrcmp(record.comment, r_out.comment) != 0) + TEST_ERROR; + + if (H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR != r_out.archival_index.version) + TEST_ERROR; + if (record.archival_index.page_size_log2 != r_out.archival_index.page_size_log2) + TEST_ERROR; + if (record.archival_index.n_entries != r_out.archival_index.n_entries) + TEST_ERROR; + for (i = 0; i < record.archival_index.n_entries; i++) { + H5FD_onion_index_entry_t *ep = &record.archival_index.list[i]; + H5FD_onion_index_entry_t *ap = &r_out.archival_index.list[i]; + + if (ep->phys_addr != ap->phys_addr) + TEST_ERROR; + if (ep->logical_page != ap->logical_page) + TEST_ERROR; + } + + /* Cleanup */ + + HDfree(r_out.archival_index.list); + HDfree(r_out.comment); + HDfree(buf); + HDfree(record.archival_index.list); + + PASSED(); + return 0; + +error: + HDfree(r_out.archival_index.list); + HDfree(r_out.comment); + HDfree(buf); + HDfree(record.archival_index.list); + + return -1; +} /* end test_revision_record_encode_decode() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * Use VFL to open target file and check that its bytes exactly match those + * of given buffer 'exp'[ected]. + * + * Returns 0 if successful, -1 if error or mismatch. + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static int +compare_file_bytes_exactly(const char *filepath, hid_t fapl_id, size_t nbytes, const unsigned char *exp) +{ + H5FD_t *raw_vfile = NULL; /* virtual file to look at raw file contents */ + unsigned char *act_buf = NULL; /* allocated area for actual file bytes */ + uint64_t filesize = 0; + + if (NULL == (raw_vfile = H5FDopen(filepath, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF))) + TEST_ERROR; + + /* filesize is wrong w/ stdio - it's zero instead of 40 or whatnot */ + filesize = (uint64_t)H5FDget_eof(raw_vfile, H5FD_MEM_DRAW); + if ((uint64_t)nbytes != filesize) { + HDfprintf(stderr, "\nSizes not the same - nbytes: %zu, filesize: %" PRIu64 "\n", nbytes, filesize); + TEST_ERROR; + } + + if (NULL == (act_buf = HDmalloc(nbytes))) + TEST_ERROR; + /* Fill buffer with bogus UCHAR_MAX values */ + for (size_t i = 0; i < nbytes; i++) + act_buf[i] = UCHAR_MAX; + if (H5FDset_eoa(raw_vfile, H5FD_MEM_DRAW, nbytes) < 0) + TEST_ERROR; + if (H5FDread(raw_vfile, H5FD_MEM_DRAW, H5P_DEFAULT, 0, nbytes, act_buf) < 0) + TEST_ERROR; + + /* Compare raw bytes data */ + for (size_t i = 0; i < nbytes; i++) { + if (exp[i] != act_buf[i]) { + HDprintf("first mismatched byte %zu: expected 0x%02X was 0x%02X\n", i, exp[i], act_buf[i]); + TEST_ERROR; + } + } + + if (H5FDclose(raw_vfile) < 0) + TEST_ERROR; + HDfree(act_buf); + + return 0; + +error: + if (raw_vfile != NULL) + H5FDclose(raw_vfile); + HDfree(act_buf); + + return -1; +} /* end compare_file_bytes_exactly() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * Do a manual read of the onion history (separate, single "Onion" file). + * Verify that the history data is well-formed and matches the expected state. + * + * Inspect file contents on backing store. + * Return -1 on problem, 0 if okay. + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static int +verify_history_as_expected_onion(H5FD_t *raw_file, struct expected_history *filter) +{ + unsigned char *buf = NULL; /* allocated area for actual file bytes */ + H5FD_onion_header_t hdr_out; + H5FD_onion_history_t history_out; + H5FD_onion_revision_record_t rev_out; + uint64_t filesize = 0; + uint64_t readsize = 0; + uint8_t *ui8p = NULL; + uint32_t buf_checksum = 0; + + /* memset to avoid bad frees on errors */ + HDmemset(&rev_out, 0, sizeof(H5FD_onion_revision_record_t)); + HDmemset(&history_out, 0, sizeof(H5FD_onion_history_t)); + + hdr_out.version = H5FD_ONION_HEADER_VERSION_CURR; + + history_out.version = H5FD_ONION_HISTORY_VERSION_CURR; + history_out.n_revisions = 0; + history_out.record_locs = NULL; + + rev_out.version = H5FD_ONION_REVISION_RECORD_VERSION_CURR; + rev_out.archival_index.version = H5FD_ONION_ARCHIVAL_INDEX_VERSION_CURR; + + filesize = (uint64_t)H5FDget_eof(raw_file, H5FD_MEM_DRAW); + if (H5FDset_eoa(raw_file, H5FD_MEM_DRAW, filesize) < 0) + TEST_ERROR; + + /* Ingest onion header */ + + readsize = MIN(filesize, H5FD_ONION_ENCODED_SIZE_HEADER); + if (NULL == (buf = HDmalloc(readsize * sizeof(unsigned char)))) + TEST_ERROR; + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, readsize, buf) < 0) + TEST_ERROR; + + readsize = H5FD__onion_header_decode(buf, &hdr_out); + if (0 == readsize) + TEST_ERROR; + if (H5FD_ONION_HEADER_VERSION_CURR != hdr_out.version) + TEST_ERROR; + /* Decode from the buffer to we can compare on BE systems */ + ui8p = (uint8_t *)(&buf[readsize - 4]); + UINT32DECODE(ui8p, buf_checksum); + if (hdr_out.checksum != buf_checksum) + TEST_ERROR; + if (hdr_out.checksum != H5_checksum_fletcher32(buf, readsize - 4)) + TEST_ERROR; + if (filter->page_size != hdr_out.page_size) + TEST_ERROR; + if (hdr_out.history_addr + hdr_out.history_size != filesize) + TEST_ERROR; + if (filter->origin_eof != hdr_out.origin_eof) + TEST_ERROR; + + HDfree(buf); + buf = NULL; + + /* Ingest history */ + + readsize = hdr_out.history_size; + if (NULL == (buf = HDmalloc(readsize * sizeof(unsigned char)))) + TEST_ERROR; + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, hdr_out.history_addr, readsize, buf) < 0) + TEST_ERROR; + + /* Initial read, get count of revisions */ + readsize = H5FD__onion_history_decode(buf, &history_out); + if (0 == readsize) + TEST_ERROR; + if (H5FD_ONION_HISTORY_VERSION_CURR != history_out.version) + TEST_ERROR; + /* Decode from the buffer to we can compare on BE systems */ + ui8p = (uint8_t *)(&buf[readsize - 4]); + UINT32DECODE(ui8p, buf_checksum); + if (history_out.checksum != buf_checksum) + TEST_ERROR; + if (history_out.checksum != H5_checksum_fletcher32(buf, readsize - 4)) + TEST_ERROR; + if (filter->n_revisions != history_out.n_revisions) + TEST_ERROR; + + /* Final read, populate pointers to revision records */ + history_out.record_locs = HDcalloc(history_out.n_revisions, sizeof(H5FD_onion_record_loc_t)); + if (NULL == history_out.record_locs) + TEST_ERROR; + if (H5FD__onion_history_decode(buf, &history_out) != readsize) + TEST_ERROR; + + /* Re-use buffer space to sanity-check checksum for record pointer(s). */ + HDassert(readsize >= sizeof(H5FD_onion_record_loc_t)); + for (size_t i = 0; i < history_out.n_revisions; i++) { + + uint64_t phys_addr; + uint64_t record_size; + + /* Do a checked assignment from the struct value into appropriately + * sized types. We don't have access to the H5F_t struct for this + * file, so we can't use the offset/length macros in H5Fprivate.h. + * + * Have to do an encode to get these values so the test passes on BE + * systems. + */ + H5_CHECKED_ASSIGN(phys_addr, uint64_t, history_out.record_locs[i].phys_addr, haddr_t); + H5_CHECKED_ASSIGN(record_size, uint64_t, history_out.record_locs[i].record_size, hsize_t); + + ui8p = (uint8_t *)buf; + UINT64ENCODE(ui8p, phys_addr); + + ui8p = (uint8_t *)(buf + 8); + UINT64ENCODE(ui8p, record_size); + + if (history_out.record_locs[i].checksum != H5_checksum_fletcher32(buf, 16)) + TEST_ERROR; + } + + HDfree(buf); + buf = NULL; + + /* Ingest revision(s) */ + + for (size_t i = 0; i < history_out.n_revisions; i++) { + H5FD_onion_record_loc_t *rpp = &history_out.record_locs[i]; + struct expected_revision *erp = &filter->revisions[i]; + + rev_out.archival_index.list = NULL; + rev_out.archival_index.n_entries = 0; + rev_out.archival_index.page_size_log2 = 0; + rev_out.comment_size = 0; + rev_out.comment = NULL; + + readsize = rpp->record_size; + if (NULL == (buf = HDmalloc((size_t)rpp->record_size))) + TEST_ERROR; + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, rpp->phys_addr, rpp->record_size, buf) < 0) + TEST_ERROR; + + /* Initial revision read -- get fixed components */ + readsize = H5FD__onion_revision_record_decode(buf, &rev_out); + if (0 == readsize) + TEST_ERROR; + if (rpp->record_size != readsize) + TEST_ERROR; + if (H5FD_ONION_REVISION_RECORD_VERSION_CURR != rev_out.version) + TEST_ERROR; + /* Decode from the buffer to we can compare on BE systems */ + ui8p = (uint8_t *)(&buf[readsize - 4]); + UINT32DECODE(ui8p, buf_checksum); + if (rev_out.checksum != buf_checksum) + TEST_ERROR; + if (rev_out.checksum != H5_checksum_fletcher32(buf, readsize - 4)) + TEST_ERROR; + if (erp->revision_num != rev_out.revision_num) + TEST_ERROR; + if (erp->parent_revision_num != rev_out.parent_revision_num) + TEST_ERROR; + if (erp->logical_eof != rev_out.logical_eof) + TEST_ERROR; + + /* Final read, get variable-length data */ + if (NULL == (rev_out.comment = HDmalloc((size_t)rev_out.comment_size))) + TEST_ERROR; + rev_out.archival_index.list = + HDcalloc(rev_out.archival_index.n_entries, sizeof(H5FD_onion_index_entry_t)); + if (NULL == rev_out.archival_index.list) + TEST_ERROR; + + readsize = H5FD__onion_revision_record_decode(buf, &rev_out); + if (rpp->record_size != readsize) + TEST_ERROR; + + /* Compare revision info with expected filter */ + if (erp->comment == NULL) { + if (rev_out.comment_size != 0) + TEST_ERROR; + } + else { + if (HDstrlen(rev_out.comment) != HDstrlen(erp->comment)) + TEST_ERROR; + if (HDstrcmp(rev_out.comment, erp->comment) != 0) + TEST_ERROR; + } + if (erp->n_index_entries != (uint64_t)(-1) && + erp->n_index_entries != rev_out.archival_index.n_entries) + TEST_ERROR; + + HDfree(buf); + HDfree(rev_out.comment); + HDfree(rev_out.archival_index.list); + } + + HDfree(history_out.record_locs); + history_out.record_locs = NULL; + + return 0; + +error: + HDfree(buf); + HDfree(rev_out.comment); + HDfree(rev_out.archival_index.list); + HDfree(history_out.record_locs); + + return -1; + +} /* end verify_history_as_expected_onion() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * Verify file bytes on the backing store + * + onion storage target + * + create from nothing + * + stage 0 (initializing) + * + open (not yet written) + * + "Empty" .h5 file created + * + .onion file created w/ only header (0 whole-hist addr) + * + .onion.recovery created w/ "empty" history + * + Cannot open onionized canonical file (incomplete history, no rev) + * + * Inspect file contents on backing store. + * Return -1 on problem, 0 if okay. + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static int +verify_stored_onion_create_0_open(struct onion_filepaths *paths, H5FD_onion_fapl_info_t *onion_info) +{ + H5FD_t *file = NULL; /* virtual file to look at raw file contents */ + unsigned char *act_buf = NULL; /* allocated area for actual file bytes */ + hid_t fapl_id = onion_info->backing_fapl_id; + herr_t err_ret = FAIL; + unsigned char hdr_exp_bytes[] = { + 'O', 'H', 'D', 'H', 1, 1, 0, 0, 0, 0, 0, 0, /* page-size encoded below */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* checksum encoded below */ + }; + size_t history_exp_bytes_size = 20; + unsigned char history_exp_bytes[] = { + 'O', 'W', 'H', 'S', 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* checksum encoded below */ + }; + unsigned char *ptr = NULL; + uint32_t checksum = 0; + + /* Finish populating expected header bytes */ + ptr = hdr_exp_bytes + 8; /* WARNING: must match format */ + UINT32ENCODE(ptr, onion_info->page_size); + checksum = H5_checksum_fletcher32(hdr_exp_bytes, H5FD_ONION_ENCODED_SIZE_HEADER - 4); + ptr = hdr_exp_bytes + H5FD_ONION_ENCODED_SIZE_HEADER - 4; + UINT32ENCODE(ptr, checksum); + ptr = NULL; + + /* Finish populating expected history bytes */ + checksum = H5_checksum_fletcher32(history_exp_bytes, H5FD_ONION_ENCODED_SIZE_HISTORY - 4); + ptr = history_exp_bytes + H5FD_ONION_ENCODED_SIZE_HISTORY - 4; + UINT32ENCODE(ptr, checksum); + ptr = NULL; + + /* Look at h5 file: should have zero bytes */ + + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + + /* Size here is arbitrary */ + if (NULL == (act_buf = HDcalloc(1, 8))) + TEST_ERROR; + + /* Should fail when reading from an empty file */ + H5E_BEGIN_TRY + { + err_ret = H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 1, act_buf); + } + H5E_END_TRY; + if (err_ret != FAIL) + TEST_ERROR; + + HDfree(act_buf); + act_buf = NULL; + + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + + /* Look at onion file: should have header */ + if (compare_file_bytes_exactly(paths->onion, fapl_id, H5FD_ONION_ENCODED_SIZE_HEADER, hdr_exp_bytes) < 0) + TEST_ERROR; + + /* Look at history backing file: should have nascent history */ + if (compare_file_bytes_exactly(paths->recovery, fapl_id, history_exp_bytes_size, history_exp_bytes) < 0) + TEST_ERROR; + + /* Inspect .h5 file contents */ + if (compare_file_bytes_exactly(paths->canon, fapl_id, 8, (const unsigned char *)"ONIONEOF") < 0) + TEST_ERROR; + + return 0; + +error: + if (file != NULL) + (void)H5FDclose(file); + HDfree(act_buf); + + return -1; +} /* end verify_stored_onion_create_0_open() */ + +/*----------------------------------------------------------------------------- + * Function: test_create_oniontarget() + * + * Purpose: Test the ability of the Onion VFD to create a valid + * 'onionized' file. + * + * When `truncate_canonical` is FALSE, the canonical file is + * nonexistent on the backing store on onion-creation. + * When `truncate_canonical` is TRUE, a canonical file is created + * on the backing store with known contents, which are to be + * truncated on onion-creation. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_create_oniontarget(hbool_t truncate_canonical, hbool_t with_initial_data) +{ + const char *basename = "somesuch"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "initial commit" /* comment */ + }; + H5FD_t *vfile_raw = NULL; /* virtual file to look at raw file contents */ + H5FD_t *vfile_rw = NULL; /* Onion virtual file for read/write */ + H5FD_t *vfile_ro = NULL; /* Onion virtual file for read-only */ + struct expected_history filter; + char *buf = NULL; + + if (TRUE == truncate_canonical && TRUE == with_initial_data) + TESTING("onion creation; truncate extant canonical; w/ initial data"); + else if (TRUE == truncate_canonical) + TESTING("onion creation; truncate extant canonical; no initial data"); + else if (TRUE == with_initial_data) + TESTING("onion creation; no extant canonical; w/ initial data"); + else + TESTING("onion creation; no extant canonical; no initial data"); + + /********* + * SETUP * + *********/ + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /* Create canonical file to be truncated */ + if (TRUE == truncate_canonical) { + /* Create canonical file. */ + vfile_raw = H5FDopen(paths->canon, flags_create_s, onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == vfile_raw) + TEST_ERROR; + if (H5FDset_eoa(vfile_raw, H5FD_MEM_DRAW, b_list_size_s) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_raw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, b_list_size_s, b_list_s) < 0) + TEST_ERROR; + if (H5FDclose(vfile_raw) < 0) + TEST_ERROR; + + vfile_raw = NULL; + H5E_BEGIN_TRY + { + vfile_raw = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + } + H5E_END_TRY; + + /* Check if onion history for onion-open created file */ + if (NULL != vfile_raw) + TEST_ERROR; + + /* Create "existing onion file". */ + vfile_raw = H5FDopen(paths->onion, flags_create_s, onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == vfile_raw) + TEST_ERROR; + if (H5FDset_eoa(vfile_raw, H5FD_MEM_DRAW, b_list_size_s) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_raw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 23, "prior history stand-in") < 0) + TEST_ERROR; + if (H5FDclose(vfile_raw) < 0) + TEST_ERROR; + vfile_raw = NULL; + } /* end if to create canonical file for truncation */ + + /* + * OPENED + */ + + /* Begin creation of onionized file from nothing */ + + vfile_rw = H5FDopen(paths->canon, flags_create_s, fapl_id, HADDR_UNDEF); + if (NULL == vfile_rw) + TEST_ERROR; + + if (verify_stored_onion_create_0_open(paths, &onion_info) < 0) + TEST_ERROR; + + H5E_BEGIN_TRY + { + vfile_ro = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + } + H5E_END_TRY; + /* check if onionization (creation) not complete; nothing to open */ + if (vfile_ro != NULL) + TEST_ERROR; + + /* + * WRITING + */ + + if (TRUE == with_initial_data) { + haddr_t half_size = 0; + haddr_t buf_size = 0; + + /* Write the sub-page entry at addr 0 */ + if (4 >= onion_info.page_size) + TEST_ERROR; + if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, 4) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 4, a_list_s) < 0) { + TEST_ERROR; + } + + /* Verify logical file contents. */ + if (NULL == (buf = HDmalloc(4 * sizeof(char)))) + TEST_ERROR; + if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 4, buf) < 0) + TEST_ERROR; + if (HDmemcmp(a_list_s, buf, 4) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + + /* Write the latter half of buffer at addr 0 (more than one page) */ + half_size = a_list_size_s / 2; + buf_size = a_list_size_s - half_size; + if (buf_size <= onion_info.page_size) + TEST_ERROR; + if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, buf_size) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, buf_size, a_list_s + half_size) < 0) + TEST_ERROR; + + /* Verify logical file contents. */ + if (NULL == (buf = HDmalloc(buf_size * sizeof(char)))) + TEST_ERROR; + if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, buf_size, buf) < 0) + TEST_ERROR; + if (HDmemcmp(a_list_s + half_size, buf, buf_size) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + + /* Overwrite existing data with entire buffer at addr 0 */ + if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, a_list_size_s) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, a_list_s) < 0) + TEST_ERROR; + + /* Verify logical file contents. */ + if (NULL == (buf = HDmalloc(a_list_size_s * sizeof(char)))) + TEST_ERROR; + if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, buf) < 0) + TEST_ERROR; + if (HDmemcmp(a_list_s, buf, a_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + } /* end if writing data to logical file */ + + /* + * CLOSED + */ + + if (H5FDclose(vfile_rw) < 0) + TEST_ERROR; + vfile_rw = NULL; + + /* Look at h5 file: should be known-empty */ + if (compare_file_bytes_exactly(paths->canon, onion_info.backing_fapl_id, 8, + (const unsigned char *)"ONIONEOF") < 0) + TEST_ERROR; + + /* Look at recovery file: should be gone */ + H5E_BEGIN_TRY + { + vfile_raw = H5FDopen(paths->recovery, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF); + } + H5E_END_TRY; + if (NULL != vfile_raw) + TEST_ERROR; + + /* Inspect onion file */ + vfile_raw = H5FDopen(paths->onion, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == vfile_raw) + TEST_ERROR; + + filter.page_size = onion_info.page_size; + filter.n_revisions = 1; + filter.origin_eof = 0; + filter.revisions[0].comment = onion_info.comment; + filter.revisions[0].n_index_entries = (uint64_t)(-1); /* don't care */ + filter.revisions[0].revision_num = 0; + filter.revisions[0].parent_revision_num = 0; + filter.revisions[0].logical_eof = (TRUE == with_initial_data) ? a_list_size_s : 0; + + if (verify_history_as_expected_onion(vfile_raw, &filter) < 0) + TEST_ERROR; + + if (H5FDclose(vfile_raw) < 0) + TEST_ERROR; + vfile_raw = NULL; + + /* R/O open the file with Onion VFD; inspect logial file */ + + vfile_ro = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == vfile_ro) + TEST_ERROR; + + if (TRUE == with_initial_data) { + /* Initial revision contains data */ + if (H5FDget_eof(vfile_ro, H5FD_MEM_DRAW) != a_list_size_s) + TEST_ERROR; + if (H5FDget_eoa(vfile_ro, H5FD_MEM_DRAW) != 0) + TEST_ERROR; + if (H5FDset_eoa(vfile_ro, H5FD_MEM_DRAW, a_list_size_s) < 0) + TEST_ERROR; + if (NULL == (buf = HDmalloc(a_list_size_s * 64 * sizeof(char)))) + TEST_ERROR; + if (H5FDread(vfile_ro, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, buf) < 0) + TEST_ERROR; + if (HDmemcmp(a_list_s, buf, a_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + } + else { + /* Initial revision has no data */ + if (H5FDget_eoa(vfile_ro, H5FD_MEM_DRAW) != 0) + TEST_ERROR; + if (H5FDget_eof(vfile_ro, H5FD_MEM_DRAW) != 0) + TEST_ERROR; + } + + if (H5FDclose(vfile_ro) < 0) + TEST_ERROR; + vfile_ro = NULL; + + /* + * CLEANUP + */ + + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + HDfree(buf); + + if (vfile_raw != NULL) + (void)H5FDclose(vfile_raw); + if (vfile_rw != NULL) + (void)H5FDclose(vfile_rw); + if (vfile_ro != NULL) + (void)H5FDclose(vfile_ro); + + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + } + H5E_END_TRY; + + return -1; +} /* end test_create_oniontarget() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_several_revisions_with_logical_gaps() + * + * Purpose: Test the ability of the Onion VFD to create a valid + * 'onionized' file. + * + * When `truncate_canonical` is FALSE, the canonical file is + * nonexistent on the backing store on onion-creation. + * When `truncate_canonical` is TRUE, a canonical file is created + * on the backing store with known contents, which are to be + * truncated on onion-creation. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_several_revisions_with_logical_gaps(void) +{ + const char *basename = "somesuch"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* flags */ + "first" /* comment */ + }; + H5FD_t *file = NULL; /* Onion virtual file for read/write */ + struct expected_history filter; + unsigned char *buf = NULL; + struct revise_revision about[4]; + H5FD_onion_history_t history_out; + size_t i = 0; + haddr_t size = 0; + uint64_t a_off = ONION_TEST_PAGE_SIZE_5 + 7; /* 39 */ + uint64_t b_off = (((a_off + a_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5) << 5) + + ONION_TEST_PAGE_SIZE_5 + 7; /* full page between */ + + TESTING("multiple revisions with gaps and overlap"); + + /********* + * SETUP * + *********/ + + history_out.version = H5FD_ONION_HISTORY_VERSION_CURR; + history_out.n_revisions = 0; + history_out.record_locs = NULL; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /* Empty first revision */ + about[0].truncate = TRUE; + about[0].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[0].comment = "first"; + about[0].n_writes = 0; + + about[1].truncate = FALSE; + about[1].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[1].comment = "second"; + about[1].n_writes = 1; + about[1].writes[0].offset = a_off; + about[1].writes[0].size = a_list_size_s; + about[1].writes[0].buf = a_list_s; + + about[2].truncate = FALSE; + about[2].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[2].comment = "third"; + about[2].n_writes = 1; /* TODO: several writes */ + about[2].writes[0].offset = b_off; + about[2].writes[0].size = b_list_size_s; + about[2].writes[0].buf = b_list_s; + + about[3].truncate = FALSE; + about[3].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[3].comment = "fourth"; + about[3].n_writes = 1; + about[3].writes[0].offset = 0; + about[3].writes[0].size = a_list_size_s; + about[3].writes[0].buf = a_list_s; + + if (do_onion_open_and_writes(paths->canon, &onion_info, 4, about) < 0) + TEST_ERROR; + + /* Inspect logical file */ + + /* THIS IS THE INITIAL FILE, SHOULD ONLY HAVE 8 BYTES */ + onion_info.revision_num = 0; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + if (8 != H5FDget_eof(file, H5FD_MEM_DRAW)) { + HDprintf("\nEOF is not zero, it is: %" PRIuHADDR "\n", H5FDget_eof(file, H5FD_MEM_DRAW)); + TEST_ERROR; + } + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* Empty first revision */ + onion_info.revision_num = 1; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + if (0 != H5FDget_eof(file, H5FD_MEM_DRAW)) { + HDprintf("\nEOF is not zero, it is: %" PRIuHADDR "\n", H5FDget_eof(file, H5FD_MEM_DRAW)); + TEST_ERROR; + } + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* One offset block in second revision */ + onion_info.revision_num = 2; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + size = a_off + a_list_size_s; + if (size != H5FDget_eof(file, H5FD_MEM_DRAW)) { + HDprintf("\nEOF is not %" PRIuHADDR ", it is: %" PRIuHADDR "\n", size, + H5FDget_eof(file, H5FD_MEM_DRAW)); + TEST_ERROR; + } + if (NULL == (buf = HDmalloc(size * sizeof(unsigned char)))) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0) + TEST_ERROR; + for (i = 0; i < a_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + a_off, a_list_s, a_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + /* Repeat read at page offset; test possible read offset error */ + if (NULL == (buf = HDmalloc(ONION_TEST_PAGE_SIZE_5 * sizeof(unsigned char)))) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, ONION_TEST_PAGE_SIZE_5, ONION_TEST_PAGE_SIZE_5, buf) < 0) + TEST_ERROR; + size = a_off - ONION_TEST_PAGE_SIZE_5; + for (i = 0; i < size; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + size, a_list_s, ONION_TEST_PAGE_SIZE_5 - size) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* Two offset blocks in third revision */ + onion_info.revision_num = 3; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + size = b_off + b_list_size_s; + if (size != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + if (NULL == (buf = HDmalloc(size * sizeof(unsigned char)))) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0) + TEST_ERROR; + for (i = 0; i < a_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + a_off, a_list_s, a_list_size_s) != 0) + TEST_ERROR; + for (i = a_off + a_list_size_s; i < b_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + b_off, b_list_s, b_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* From start and partial overwrite in fourth revision */ + onion_info.revision_num = 4; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + size = b_off + b_list_size_s; + if (size != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * size); + if (NULL == buf) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0) + TEST_ERROR; + if (HDmemcmp(buf, a_list_s, a_list_size_s) != 0) + TEST_ERROR; + if (HDmemcmp(buf + a_list_size_s, a_list_s + a_list_size_s - a_off, a_off) != 0) + TEST_ERROR; + for (i = a_off + a_list_size_s; i < b_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + b_off, b_list_s, b_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* No fifth revision */ + + /* Inspect history construction */ + + file = H5FDopen(paths->onion, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + + filter.page_size = onion_info.page_size; + filter.n_revisions = 4; + filter.origin_eof = 0; + + filter.revisions[0].comment = "first"; + filter.revisions[0].n_index_entries = 0; + filter.revisions[0].revision_num = 0; + filter.revisions[0].parent_revision_num = 0; + filter.revisions[0].logical_eof = 0; + + filter.revisions[1].comment = "second"; + filter.revisions[1].n_index_entries = (a_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5; + filter.revisions[1].revision_num = 1; + filter.revisions[1].parent_revision_num = 0; + filter.revisions[1].logical_eof = a_off + a_list_size_s; + + filter.revisions[2].comment = "third"; + filter.revisions[2].n_index_entries = + filter.revisions[1].n_index_entries + ((b_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5); + filter.revisions[2].revision_num = 2; + filter.revisions[2].parent_revision_num = 1; + filter.revisions[2].logical_eof = b_off + b_list_size_s; + + filter.revisions[3].comment = "fourth"; + filter.revisions[3].n_index_entries = filter.revisions[2].n_index_entries + 1; + filter.revisions[3].revision_num = 3; + filter.revisions[3].parent_revision_num = 2; + filter.revisions[3].logical_eof = b_off + b_list_size_s; + + if (verify_history_as_expected_onion(file, &filter) < 0) + TEST_ERROR; + + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + + /* CLEANUP */ + + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + HDfree(history_out.record_locs); + HDfree(buf); + + if (file != NULL) + (void)H5FDclose(file); + + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} /* end test_several_revisions_with_logical_gaps() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * + * Function: do_onion_open_and_writes + * + * Purpose: Automate the process of creating/opening a file and performing + * a series of writes. + * + * Return: Success : 0 + * Failure : -1 + * + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +// TODO: Modify to create initial file without onion +static int +do_onion_open_and_writes(const char *filename, H5FD_onion_fapl_info_t *onion_info_p, size_t n_ops, + struct revise_revision *about) +{ + hid_t fapl_id = H5I_INVALID_HID; + H5FD_t *file = NULL; /* Onion virtual file for read/write */ + unsigned char *buf_vfy = NULL; + size_t i = 0; + + for (i = 0; i < n_ops; i++) { + size_t j = 0; + unsigned int flags = H5F_ACC_RDWR; + + if (i != 0 && about[i].truncate == TRUE) + goto error; + + if (TRUE == about[i].truncate) + flags |= H5F_ACC_CREAT | H5F_ACC_TRUNC; + + onion_info_p->revision_num = about[i].revision_num; + if (about[i].comment != NULL) { + j = MIN(HDstrlen(about[i].comment), H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN); + HDmemcpy(onion_info_p->comment, about[i].comment, j); + } + onion_info_p->comment[j] = '\0'; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + goto error; + if (H5Pset_fapl_onion(fapl_id, onion_info_p) < 0) + goto error; + file = H5FDopen(filename, flags, fapl_id, HADDR_UNDEF); + if (NULL == file) + goto error; + + for (j = 0; j < about[i].n_writes; j++) { + struct write_info *wi = &about[i].writes[j]; + + /* Write to file */ + if (H5FDget_eoa(file, H5FD_MEM_DRAW) < wi->offset + wi->size && + H5FDset_eoa(file, H5FD_MEM_DRAW, wi->offset + wi->size) < 0) + TEST_ERROR; + if (H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, wi->offset, wi->size, wi->buf) < 0) + TEST_ERROR; + /* Verify write as expected */ + if (NULL == (buf_vfy = HDmalloc(wi->size * sizeof(unsigned char)))) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, wi->offset, wi->size, buf_vfy) < 0) + TEST_ERROR; + if (HDmemcmp(buf_vfy, wi->buf, wi->size) != 0) { + const unsigned char *_buf = wi->buf; + size_t z = 0; + HDputs("i exp act"); + for (z = 0; z < wi->size; z++) + HDprintf("%02zx %c %c\n", z, _buf[z], buf_vfy[z]); + HDfflush(stdout); + TEST_ERROR; + } + HDfree(buf_vfy); + buf_vfy = NULL; + } /* end for each write */ + + if (H5FDclose(file) < 0) + goto error; + file = NULL; + if (H5Pclose(fapl_id) < 0) + goto error; + fapl_id = H5I_INVALID_HID; + } /* end for each open-close cycle */ + + return 0; + +error: + if (file != NULL) + (void)H5FDclose(file); + + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + } + H5E_END_TRY; + + HDfree(buf_vfy); + + return -1; +} /* end do_onion_open_and_writes() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_page_aligned_history_create() + * + * Purpose: Verify that, when specified in FAPL on onionization/creation, + * All history writes are aligned to page-size boundaries. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_page_aligned_history_create(void) +{ + const char *basename = "somesuch"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT, + "initial commit" /* comment */ + }; + H5FD_t *file = NULL; /* Onion virtual file for read/write */ + unsigned char *buf = NULL; + struct revise_revision about[2]; + H5FD_onion_header_t hdr_out; + H5FD_onion_history_t history_out; + size_t i = 0; + uint64_t a_off = b_list_size_s - a_list_size_s; + + TESTING("page-aligned history on onion-created file"); + + /********* + * SETUP * + *********/ + + hdr_out.version = H5FD_ONION_HEADER_VERSION_CURR; + history_out.version = H5FD_ONION_HISTORY_VERSION_CURR; + history_out.n_revisions = 0; + history_out.record_locs = NULL; + + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + about[0].truncate = TRUE; + about[0].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[0].comment = "initial_commit"; + about[0].n_writes = 1; + about[0].writes[0].offset = 0; + about[0].writes[0].size = b_list_size_s; + about[0].writes[0].buf = b_list_s; + + about[1].truncate = FALSE; + about[1].revision_num = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[1].comment = "second"; + about[1].n_writes = 1; + about[1].writes[0].offset = a_off; + about[1].writes[0].size = a_list_size_s; + about[1].writes[0].buf = a_list_s; + + if (do_onion_open_and_writes(paths->canon, &onion_info, 2, about) < 0) + TEST_ERROR; + + /* Inspect logical file */ + if (NULL == (buf = HDmalloc(b_list_size_s * sizeof(unsigned char)))) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + if (b_list_size_s != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, b_list_size_s) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, b_list_size_s, buf) < 0) + TEST_ERROR; + if (HDmemcmp(a_list_s, buf + a_off, a_list_size_s) != 0) { + size_t k; + HDprintf("aoff: %" PRIu64 "\n", a_off); + HDputs("i exp act"); + for (k = 0; k < b_list_size_s; k++) { + HDprintf("%3zu:: %c : %c\n", k, (k < a_off) ? ' ' : a_list_s[k - a_off], buf[k]); + } + HDfflush(stdout); + TEST_ERROR; + } + if (HDmemcmp(b_list_s, buf, a_off) != 0) { + size_t k; + HDprintf("aoff: %" PRIu64 "\n", a_off); + HDputs("i exp act"); + for (k = 0; k < b_list_size_s; k++) { + HDprintf("%3zu:: %c : %c\n", k, (k < a_off) ? b_list_s[k] : ' ', buf[k]); + } + HDfflush(stdout); + TEST_ERROR; + } + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + HDfree(buf); + buf = NULL; + + /* Inspect history construction */ + + if (NULL == (file = H5FDopen(paths->onion, H5F_ACC_RDONLY, onion_info.backing_fapl_id, HADDR_UNDEF))) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, H5FDget_eof(file, H5FD_MEM_DRAW)) < 0) + TEST_ERROR; + + if (NULL == (buf = HDmalloc(H5FD_ONION_ENCODED_SIZE_HEADER))) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, H5FD_ONION_ENCODED_SIZE_HEADER, buf) < 0) + TEST_ERROR; + if (H5FD__onion_header_decode(buf, &hdr_out) != H5FD_ONION_ENCODED_SIZE_HEADER) + TEST_ERROR; + if (hdr_out.history_addr & ((1 << 5) - 1)) /* 5::PAGE_SIZE_5 */ + TEST_ERROR; + HDfree(buf); + buf = NULL; + + if (NULL == (buf = HDmalloc(hdr_out.history_size))) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, hdr_out.history_addr, hdr_out.history_size, buf) < 0) + TEST_ERROR; + if (H5FD__onion_history_decode(buf, &history_out) != hdr_out.history_size) + TEST_ERROR; + if (history_out.n_revisions != 2) + TEST_ERROR; + history_out.record_locs = HDcalloc(history_out.n_revisions, sizeof(H5FD_onion_record_loc_t)); + if (NULL == history_out.record_locs) + TEST_ERROR; + if (H5FD__onion_history_decode(buf, &history_out) != hdr_out.history_size) + TEST_ERROR; + HDfree(buf); + buf = NULL; + + for (i = 0; i < history_out.n_revisions; i++) { + H5FD_onion_record_loc_t *rloc = &history_out.record_locs[i]; + if (rloc->phys_addr & ((1 << 5) - 1)) /* 5::PAGE_SIZE_5 */ + TEST_ERROR; + /* TODO: check phys_addr of each page entry? */ + } + + HDfree(history_out.record_locs); + history_out.record_locs = NULL; + + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + + /* CLEANUP */ + + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + HDfree(buf); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + HDfree(history_out.record_locs); + HDfree(buf); + + if (file != NULL) + (void)H5FDclose(file); + + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} /* end test_page_aligned_history_create() */ + +/*----------------------------------------------------------------------------- + * Function: test_integration_create() + * + * Purpose: Create and make multiple revisions in an HDF5 file. + * + * Return: PASSED : 0 + * FAILED : -1 + *----------------------------------------------------------------------------- + */ +static int +test_integration_create(void) +{ + const char *basename = "integration_2d.h5"; + hid_t fapl_id = H5I_INVALID_HID; + hid_t file_id = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hsize_t dims[2] = {128, 256}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk[2] = {4, 4}; + int fillval; + struct { + int arr[128][256]; + } *wdata = NULL; + struct { + int arr[128][256]; + } *rdata = NULL; + struct { + int arr[128][256]; + } *dset_data = NULL; + + TESTING("onion-created two dimensional HDF5 file with revisions"); + + /* SETUP */ + + if (NULL == (wdata = HDcalloc(1, sizeof(*wdata)))) + TEST_ERROR; + if (NULL == (rdata = HDcalloc(1, sizeof(*rdata)))) + TEST_ERROR; + if (NULL == (dset_data = HDcalloc(1, sizeof(*dset_data)))) + TEST_ERROR; + + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + + /* Initialize data */ + for (int i = 0; i < 128; i++) + for (int j = 0; j < 256; j++) + wdata->arr[i][j] = i * j - j; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace with unlimited dimensions */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + TEST_ERROR; + + /* Create the dataset creation property list, and set the chunk + * size + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + TEST_ERROR; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + TEST_ERROR; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + /* Close and release resources */ + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Sclose(space) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < 128; i++) + for (int j = 0; j < 256; j++) + dset_data->arr[i][j] = i * 6 + j + 1; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < 128; i++) + for (int j = 0; j < 256; j++) + dset_data->arr[i][j] = i * 3 + j + 5; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Start to verify the revision + *---------------------------------------------------------------------- + */ + /*---------------------------------------------------------------------- + * Verify the original file + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 0; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0) + TEST_ERROR; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < 128; i++) { + for (int j = 0; j < 256; j++) { + int expected = i * j - j; + if (rdata->arr[i][j] != expected) { + HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i][j]); + HDfflush(stdout); + TEST_ERROR; + } + } + } + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Verify the first revision + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 1; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < 128; i++) { + for (int j = 0; j < 256; j++) { + int expected = i * 6 + j + 1; + if (rdata->arr[i][j] != expected) { + HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i][j]); + HDfflush(stdout); + TEST_ERROR; + } + } + } + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Verify the second revision + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 2; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < 128; i++) { + for (int j = 0; j < 256; j++) { + int expected = i * 3 + j + 5; + if (rdata->arr[i][j] != expected) { + HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i][j]); + HDfflush(stdout); + TEST_ERROR; + } + else { + HDfflush(stdout); + } + } + } + + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + HDfree(wdata); + HDfree(rdata); + HDfree(dset_data); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file_id); + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + HDfree(wdata); + HDfree(rdata); + HDfree(dset_data); + + return -1; +} /* end test_integration_create() */ + +static int +test_integration_create_simple(void) +{ + const char *basename = "integration_1d.h5"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hid_t file_id = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[2] = {1, ONE_DIM_SIZE}; + hsize_t maxdims[2] = {1, ONE_DIM_SIZE}; + int fillval; + struct { + int arr[ONE_DIM_SIZE]; + } *wdata = NULL; + struct { + int arr[ONE_DIM_SIZE]; + } *rdata = NULL; + struct { + int arr[ONE_DIM_SIZE]; + } *dset_data = NULL; + + TESTING("onion-created one-dimensional HDF5 file with revisions"); + + /* Setup */ + if (NULL == (wdata = HDcalloc(1, sizeof(*wdata)))) + TEST_ERROR; + if (NULL == (rdata = HDcalloc(1, sizeof(*rdata)))) + TEST_ERROR; + if (NULL == (dset_data = HDcalloc(1, sizeof(*dset_data)))) + TEST_ERROR; + + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + /* Initialize data */ + for (int i = 0; i < ONE_DIM_SIZE; i++) + wdata->arr[i] = i; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace with unlimited dimensions*/ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + TEST_ERROR; + + /* Create the dataset creation property list */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + TEST_ERROR; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + /* Close everything */ + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Sclose(space) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data->arr[i] = i + ONE_DIM_SIZE; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data->arr[i] = i + 2048; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Third revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i += 20) + dset_data->arr[i] = i + 3072; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Start to verify the revision + *---------------------------------------------------------------------- + */ + /*---------------------------------------------------------------------- + * Verify the second revision + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 2; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i += 20) { + int expected = i + 2048; + if (rdata->arr[i] != expected) { + HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i]); + TEST_ERROR; + } + } + + /* Close everything */ + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + HDfree(wdata); + HDfree(rdata); + HDfree(dset_data); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file_id); + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + HDfree(wdata); + HDfree(rdata); + HDfree(dset_data); + + return -1; +} /* end test_integration_create_simple() */ + +static int +test_integration_create_delete_objects(void) +{ + const char *basename = "integration_objs.h5"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hid_t group_id = H5I_INVALID_HID; + hid_t attr_space_id = H5I_INVALID_HID; + hid_t attr_id = H5I_INVALID_HID; + hsize_t attr_dim[1] = {4}; + + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[2] = {4, 4}, maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}, chunk[2] = {4, 4}; + int wdata[4][4], /* Write buffer */ + fillval, i, j; + + TESTING("onion-created HDF5 file with revisions testing addition and deletion of objects"); + + /* Set up */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + + /* Initialize data */ + for (i = 0; i < 4; i++) + for (j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace with unlimited dimensions */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + TEST_ERROR; + + /* Create the dataset creation property list, and set the chunk + * size + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + TEST_ERROR; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + TEST_ERROR; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and add a dataset (DS2) to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and remove the dataset (DS2), + * which was added during the first revision. + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Third revision: open the file with Onion VFD and add an attribute to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Create dataspace for attribute */ + if ((attr_space_id = H5Screate_simple(1, attr_dim, NULL)) < 0) + TEST_ERROR; + + if ((attr_id = + H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Sclose(attr_space_id) < 0) + TEST_ERROR; + if (H5Aclose(attr_id) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Fourth revision: open the file with Onion VFD and delete the attribute + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if (H5Adelete(file, "file_attribute") < 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Fifth revision: open the file with Onion VFD and add a group to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Gclose(group_id) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Sixth revision: open the file with Onion VFD and delete the newly added group + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Start to verify the revision + *---------------------------------------------------------------------- + */ + /*---------------------------------------------------------------------- + * Verify the first revision: it should have the second dataset (DS2) + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 1; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* The second dataset (DS2) should exist */ + if (H5Lexists(file, "DS2", H5P_DEFAULT) <= 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*------------------------------------------------------------------------ + * Verify the second revision: the second dataset (DS2) should be removed + *------------------------------------------------------------------------ + */ + onion_info.revision_num = 2; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* The second dataset (DS2) shouldn't exist */ + if (H5Lexists(file, "DS2", H5P_DEFAULT) > 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*------------------------------------------------------------------------- + * Verify the third revision: the file attribute (file_attribute) should exist + *------------------------------------------------------------------------- + */ + onion_info.revision_num = 3; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* The file attribute should exist */ + if (H5Aexists(file, "file_attribute") <= 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*------------------------------------------------------------------------- + * Verify the fourth revision: the file attribute (file_attribute) should be removed + *------------------------------------------------------------------------- + */ + onion_info.revision_num = 4; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* The file attribute should be removed */ + if (H5Aexists(file, "file_attribute") > 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*------------------------------------------------------------------------- + * Verify the fifth revision: the group (new_group) should exist + *------------------------------------------------------------------------- + */ + onion_info.revision_num = 5; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* The new group should exist */ + if (H5Lexists(file, "new_group", H5P_DEFAULT) <= 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*------------------------------------------------------------------------- + * Verify the sixth revision: the group (new_group) should be removed + *------------------------------------------------------------------------- + */ + onion_info.revision_num = 6; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* The new group should exist */ + if (H5Lexists(file, "new_group", H5P_DEFAULT) > 0) + TEST_ERROR; + + /* Close everything */ + if (H5Fclose(file) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Sclose(space) < 0) + TEST_ERROR; + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file); + H5Pclose(fapl_id); + H5Sclose(space); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} /* end test_integration_create_delete_objects */ + +static int +test_integration_dset_extension(void) +{ + const char *basename = "integration_dset_ext.h5"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset_space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[2] = {4, 4}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk[2] = {4, 4}; + hsize_t size[2]; + hsize_t offset[2]; + int wdata[4][4]; /* Write buffer */ + int fillval; + int rdata[4][4]; /* Read buffer */ + + TESTING("onion-created HDF5 file with revisions testing dataset extension"); + + /* Setup */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + + /* Initialize data */ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace with unlimited dimensions*/ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + TEST_ERROR; + + /* Create the dataset creation property list, and set the chunk + * size + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + TEST_ERROR; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + TEST_ERROR; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and extend the dataset + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Open the dataset */ + if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Extend the dataset and double the rows */ + size[0] = 2 * dims[0]; + size[1] = dims[1]; + if (H5Dset_extent(dset, size) < 0) + TEST_ERROR; + + if ((dset_space = H5Dget_space(dset)) < 0) + TEST_ERROR; + + offset[0] = dims[0]; + offset[1] = 0; + if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0) + TEST_ERROR; + + /* Write the data to the dataset. */ + if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + if (H5Sclose(dset_space) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and shrink the dataset + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Open the dataset */ + if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Extend the dataset and shrink back the size */ + if (H5Dset_extent(dset, dims) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Start to verify the revision + *---------------------------------------------------------------------- + */ + /*---------------------------------------------------------------------- + * Verify the first revision: it should have the extended data + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 1; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Open the dataset */ + if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + if ((dset_space = H5Dget_space(dset)) < 0) + TEST_ERROR; + + offset[0] = dims[0]; + offset[1] = 0; + if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0) + TEST_ERROR; + + if (H5Dread(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + if (rdata[i][j] != wdata[i][j]) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Verify the second revision: it should have the original data + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 2; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Open the dataset */ + dset = H5Dopen2(file, "DS1", H5P_DEFAULT); + + if ((dset_space = H5Dget_space(dset)) < 0) + TEST_ERROR; + + if (H5Dread(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + if (rdata[i][j] != wdata[i][j]) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /* Close and release resources. */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Sclose(space) < 0) + TEST_ERROR; + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file); + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} /* end test_integration_dset_extension */ + +static int +test_integration_ctl(void) +{ + const char *basename = "integration_ctl.h5"; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hsize_t dims[2] = {4, 4}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk[2] = {4, 4}; + int wdata[4][4]; /* Write buffer */ + int fillval; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + uint64_t revision_count; + + TESTING("onion-created HDF5 file with revisions testing H5FDctl"); + + /* Set up */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + + /* Initialize data */ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace with unlimited dimensions */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + TEST_ERROR; + + /* Create the dataset creation property list, and set the chunk size */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + TEST_ERROR; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + TEST_ERROR; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and add a dataset (DS2) to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and remove the dataset (DS2), + * which was added during the first revision. + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0) + TEST_ERROR; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Start to verify the number of revisions + *---------------------------------------------------------------------- + */ + if (H5FDonion_get_revision_count(basename, fapl_id, &revision_count) < 0) + TEST_ERROR; + + if (2 != revision_count) + TEST_ERROR; + + /* Close and release resources */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Sclose(space) < 0) + TEST_ERROR; + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file); + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} + +static int +test_integration_reference(void) +{ + const char *basename = "integration_refer.h5"; + hid_t file = H5I_INVALID_HID; + hid_t group = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t space2 = H5I_INVALID_HID; + hid_t space_ref = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dset2 = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hsize_t dims[2] = {4, 4}; + hsize_t dim_ref[1] = {2}; + int wdata[4][4]; /* Write buffer */ + int rdata[4][4]; /* Read buffer */ + H5R_ref_t wbuf[2]; + H5R_ref_t rbuf[2]; + H5O_type_t obj_type; + hsize_t start[2]; + hsize_t stride[2]; + hsize_t count[2]; + hsize_t block[2]; + hsize_t coord1[4][2]; /* Coordinates for point selection */ + hssize_t nelmts; + + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + + TESTING("onion-created HDF5 file with revisions testing references"); + + /* Set up */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + + /* Initialize data */ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if ((space = H5Screate_simple(2, dims, NULL)) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and add a dataset (DS2) + * of object references + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Create dataspace with unlimited dimensions */ + if ((space_ref = H5Screate_simple(1, dim_ref, NULL)) < 0) + TEST_ERROR; + + /* Create the dataset of object references */ + if ((dset = H5Dcreate2(file, "DS2", H5T_STD_REF, space_ref, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create reference to dataset */ + if (H5Rcreate_object(file, "DS1", H5P_DEFAULT, &wbuf[0]) < 0) + TEST_ERROR; + + if (H5Rget_obj_type3(&wbuf[0], H5P_DEFAULT, &obj_type) < 0) + TEST_ERROR; + + if (obj_type != H5O_TYPE_DATASET) + TEST_ERROR; + + /* Create reference to the root group */ + if (H5Rcreate_object(file, "/", H5P_DEFAULT, &wbuf[1]) < 0) + TEST_ERROR; + + if (H5Rget_obj_type3(&wbuf[1], H5P_DEFAULT, &obj_type) < 0) + TEST_ERROR; + + if (obj_type != H5O_TYPE_GROUP) + TEST_ERROR; + + /* Write the object reference data to the dataset */ + if (H5Dwrite(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + for (int i = 0; i < 2; i++) + if (H5Rdestroy(&wbuf[i]) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and add a dataset (DS3) + * of region references + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + /* Create the dataset of region references */ + if ((dset = H5Dcreate2(file, "DS3", H5T_STD_REF, space_ref, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Select 2x4 hyperslab for first reference */ + start[0] = 0; + start[1] = 0; + stride[0] = 1; + stride[1] = 1; + count[0] = 1; + count[1] = 1; + block[0] = 2; + block[1] = 4; + + /* Make a hyperslab selection of 2x4 elements */ + if (H5Sselect_hyperslab(space, H5S_SELECT_SET, start, stride, count, block) < 0) + TEST_ERROR; + + /* Verify the number of selection */ + if ((nelmts = H5Sget_select_npoints(space)) != 8) { + HDprintf("Number of selected elements is supposed to be 8, but got %" PRIuHSIZE "\n", nelmts); + TEST_ERROR; + } + + /* Store first data region */ + if (H5Rcreate_region(file, "/DS1", space, H5P_DEFAULT, &wbuf[0]) < 0) + TEST_ERROR; + + if (H5Rget_obj_type3(&wbuf[0], H5P_DEFAULT, &obj_type) < 0) + TEST_ERROR; + + if (obj_type != H5O_TYPE_DATASET) + TEST_ERROR; + + /* Select the sequence of four points for the second reference */ + coord1[0][0] = 0; + coord1[0][1] = 0; + coord1[1][0] = 1; + coord1[1][1] = 1; + coord1[2][0] = 2; + coord1[2][1] = 2; + coord1[3][0] = 3; + coord1[3][1] = 3; + + if (H5Sselect_elements(space, H5S_SELECT_SET, 4, (const hsize_t *)coord1) < 0) + TEST_ERROR; + + /* Store the second data region */ + if (H5Rcreate_region(file, "/DS1", space, H5P_DEFAULT, &wbuf[1]) < 0) + TEST_ERROR; + + if (H5Rget_obj_type3(&wbuf[1], H5P_DEFAULT, &obj_type) < 0) + TEST_ERROR; + + if (obj_type != H5O_TYPE_DATASET) + TEST_ERROR; + + /* Write the region reference data to the dataset */ + if (H5Dwrite(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + for (int i = 0; i < 2; i++) + if (H5Rdestroy(&wbuf[i]) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * Start to verify the revisions + *---------------------------------------------------------------------- + */ + /*---------------------------------------------------------------------- + * Verify the first revision: it should have the object references + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 1; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if ((file = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0) + TEST_ERROR; + + /* Open the dataset of the object references */ + if ((dset = H5Dopen2(file, "DS2", H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Dread(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + TEST_ERROR; + + /* Open the referenced dataset and check the data */ + if ((dset2 = H5Ropen_object(&rbuf[0], H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Dread(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + int expected = i + j; + if (rdata[i][j] != expected) { + HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata[i][j]); + TEST_ERROR; + } + } + } + + /* Open the referenced group and make sure it's a group object */ + if ((group = H5Ropen_object(&rbuf[1], H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5I_GROUP != H5Iget_type(group)) + TEST_ERROR; + + if (H5Gclose(group) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + + if (H5Dclose(dset2) < 0) + TEST_ERROR; + dset2 = H5I_INVALID_HID; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + for (int i = 0; i < 2; i++) + if (H5Rdestroy(&rbuf[i]) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * Verify the second revision: it should have the region references + *---------------------------------------------------------------------- + */ + onion_info.revision_num = 2; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + if ((file = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0) + TEST_ERROR; + + /* Open the dataset of the region reference */ + if ((dset = H5Dopen2(file, "DS3", H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Dread(dset, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + TEST_ERROR; + + /* Get the hyperslab selection and check the referenced region of the dataset */ + if ((space2 = H5Ropen_region(&rbuf[0], H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if ((nelmts = H5Sget_select_npoints(space2)) != 8) { + HDprintf("Number of selected elements is supposed to be 8, but got %" PRIuHSIZE "\n", nelmts); + TEST_ERROR; + } + + if (H5Sclose(space2) < 0) + TEST_ERROR; + space2 = H5I_INVALID_HID; + + /* Get the element selection and check the referenced region of the dataset */ + if ((space2 = H5Ropen_region(&rbuf[1], H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if ((nelmts = H5Sget_select_npoints(space2)) != 4) { + HDprintf("Number of selected elements is supposed to be 4, but got %" PRIuHSIZE "\n", nelmts); + TEST_ERROR; + } + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + + if (H5Sclose(space2) < 0) + TEST_ERROR; + space2 = H5I_INVALID_HID; + + if (H5Fclose(file) < 0) + TEST_ERROR; + file = H5I_INVALID_HID; + + for (int i = 0; i < 2; i++) + if (H5Rdestroy(&rbuf[i]) < 0) + TEST_ERROR; + + /* Close and release resources */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + if (H5Sclose(space) < 0) + TEST_ERROR; + if (H5Sclose(space_ref) < 0) + TEST_ERROR; + if (H5Pclose(onion_info.backing_fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file); + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} + +static int +test_integration_create_by_name(void) +{ + const char *basename = "integration_by_name.h5"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + hid_t file_id = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[2] = {1, ONE_DIM_SIZE}; + hsize_t maxdims[2] = {1, ONE_DIM_SIZE}; + int fillval; + struct { + int arr[ONE_DIM_SIZE]; + } *wdata = NULL; + struct { + int arr[ONE_DIM_SIZE]; + } *rdata = NULL; + struct { + int arr[ONE_DIM_SIZE]; + } *dset_data = NULL; + + TESTING("H5Pset_driver_by_name"); + + /* Setup */ + if (NULL == (wdata = HDcalloc(1, sizeof(*wdata)))) + TEST_ERROR; + if (NULL == (rdata = HDcalloc(1, sizeof(*rdata)))) + TEST_ERROR; + if (NULL == (dset_data = HDcalloc(1, sizeof(*dset_data)))) + TEST_ERROR; + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + /* Use H5Pset_driver_by_name to enable the Onion VFD */ + if (H5Pset_driver_by_name(fapl_id, "onion", "{revision_num: H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST}") < + 0) + TEST_ERROR; + + if (NULL == (paths = onion_filepaths_init(basename))) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + /* Initialize data */ + for (int i = 0; i < ONE_DIM_SIZE; i++) + wdata->arr[i] = i; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace with unlimited dimensions*/ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + TEST_ERROR; + + /* Create the dataset creation property list */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + TEST_ERROR; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + TEST_ERROR; + + /* Close everything */ + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Sclose(space) < 0) + TEST_ERROR; + if (H5Fclose(file) < 0) + TEST_ERROR; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data->arr[i] = i + ONE_DIM_SIZE; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data->arr[i] = i + 2048; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Third revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i += 20) + dset_data->arr[i] = i + 3072; + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + TEST_ERROR; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + TEST_ERROR; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Start to verify the revision with H5Pset_driver_by_name + *---------------------------------------------------------------------- + */ + /*---------------------------------------------------------------------- + * Verify the second revision + *---------------------------------------------------------------------- + */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_driver_by_name(fapl_id, "onion", "{revision_num: 2; page_size: 4; }") < 0) + TEST_ERROR; + + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id)) < 0) + TEST_ERROR; + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata) < 0) + TEST_ERROR; + + for (int i = 0; i < ONE_DIM_SIZE; i += 20) { + int expected = i + 2048; + if (rdata->arr[i] != expected) { + HDprintf("ERROR!!! Expected: %d, Got: %d\n", expected, rdata->arr[i]); + TEST_ERROR; + } + } + + /* Close everything */ + if (H5Dclose(dset) < 0) + TEST_ERROR; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + HDfree(wdata); + HDfree(rdata); + HDfree(dset_data); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file_id); + H5Pclose(fapl_id); + } + H5E_END_TRY; + + HDfree(wdata); + HDfree(rdata); + HDfree(dset_data); + + return -1; +} /* end test_integration_create_simple() */ + +/*----------------------------------------------------------------------------- + * + * Function: main() + * + * Purpose: Perform unit tests on for the Onion VFD. + * + *----------------------------------------------------------------------------- + */ +int +main(void) +{ + const char *env_h5_drvr = NULL; /* VFD value from environment */ + int nerrors = 0; + + HDprintf("Testing Onion VFD functionality.\n"); + + h5_reset(); + + /* The onion VFD only supports the sec2 VFD under the hood, so skip this + * test when the environment variable has been set to something else + */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + if ((0 != HDstrcmp(env_h5_drvr, "nomatch")) && (0 != HDstrcmp(env_h5_drvr, "sec2"))) { + SKIPPED(); + HDputs("Onion VFD test skipped due to non-sec2 default VFD"); + HDexit(EXIT_SUCCESS); + } + + /* Initialize */ + flags_create_s = H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC; + + /* Run tests. Return values on error are negative. */ + nerrors -= test_archival_index(); + nerrors -= test_revision_index(); + nerrors -= test_revision_index_collisions(); + nerrors -= test_revision_index_resizing(); + nerrors -= test_revision_index_to_archival_index(); + nerrors -= test_fapl(); + nerrors -= test_header_encode_decode(); + nerrors -= test_history_encode_decode_empty(); + nerrors -= test_history_encode_decode(); + nerrors -= test_revision_record_encode_decode(); + nerrors -= test_create_oniontarget(FALSE, FALSE); + nerrors -= test_create_oniontarget(TRUE, FALSE); + nerrors -= test_create_oniontarget(FALSE, TRUE); + nerrors -= test_create_oniontarget(TRUE, TRUE); + nerrors -= test_several_revisions_with_logical_gaps(); + nerrors -= test_page_aligned_history_create(); + nerrors -= test_integration_create(); + nerrors -= test_integration_create_simple(); + nerrors -= test_integration_create_delete_objects(); + nerrors -= test_integration_dset_extension(); + nerrors -= test_integration_ctl(); + nerrors -= test_integration_reference(); + nerrors -= test_integration_create_by_name(); + + if (nerrors > 0) { + HDprintf("***** %d Onion TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); + return EXIT_FAILURE; + } + + HDprintf("All Onion tests passed.\n"); + return EXIT_SUCCESS; + +} /* end main() */ diff --git a/tools/lib/h5diff.c b/tools/lib/h5diff.c index 7eb17ba..0bad3fe 100644 --- a/tools/lib/h5diff.c +++ b/tools/lib/h5diff.c @@ -648,8 +648,15 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char *------------------------------------------------------------------------- */ /* open file 1 */ + if (opts->vfd_info[0].u.name) { + if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[0]))) < 0) { + parallel_print("h5diff: unable to create fapl for input file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n"); + } + } + if (opts->custom_vol[0] || opts->custom_vfd[0]) { - if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL, + if ((fapl1_id = h5tools_get_fapl(fapl1_id, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL, opts->custom_vfd[0] ? &(opts->vfd_info[0]) : NULL)) < 0) { parallel_print("h5diff: unable to create fapl for input file\n"); H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n"); @@ -664,9 +671,15 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char H5TOOLS_DEBUG("file1_id = %s", fname1); /* open file 2 */ + if (opts->vfd_info[1].u.name) { + if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[1]))) < 0) { + parallel_print("h5diff: unable to create fapl for output file\n"); + H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n"); + } + } if (opts->custom_vol[1] || opts->custom_vfd[1]) { - if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL, + if ((fapl2_id = h5tools_get_fapl(fapl2_id, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL, opts->custom_vfd[1] ? &(opts->vfd_info[1]) : NULL)) < 0) { parallel_print("h5diff: unable to create fapl for output file\n"); H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n"); diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 93886d2..9c4f746 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -85,6 +85,7 @@ const char *drivernames[] = { [WINDOWS_VFD_IDX] = "windows", [STDIO_VFD_IDX] = "stdio", [CORE_VFD_IDX] = "core", [FAMILY_VFD_IDX] = "family", [SPLIT_VFD_IDX] = "split", [MULTI_VFD_IDX] = "multi", [MPIO_VFD_IDX] = "mpio", [ROS3_VFD_IDX] = "ros3", [HDFS_VFD_IDX] = "hdfs", + [ONION_VFD_IDX] = "onion", }; #define NUM_VOLS (sizeof(volnames) / sizeof(volnames[0])) @@ -573,6 +574,13 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info) H5TOOLS_GOTO_ERROR(FAIL, "The HDFS VFD is not enabled"); #endif } + else if (!HDstrcmp(vfd_info->u.name, drivernames[ONION_VFD_IDX])) { + /* Onion driver */ + if (!vfd_info->info) + H5TOOLS_GOTO_ERROR(FAIL, "Onion VFD info is invalid"); + if (H5Pset_fapl_onion(fapl_id, (const H5FD_onion_fapl_info_t *)vfd_info->info) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_onion() failed"); + } else { /* * Try to load VFD plugin. diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index 0620791..6561b68 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -600,6 +600,7 @@ typedef enum { MPIO_VFD_IDX, ROS3_VFD_IDX, HDFS_VFD_IDX, + ONION_VFD_IDX, } driver_idx; /* The following include, h5tools_str.h, must be after the diff --git a/tools/src/h5diff/h5diff_common.c b/tools/src/h5diff/h5diff_common.c index 433f0c7..87c08be 100644 --- a/tools/src/h5diff/h5diff_common.c +++ b/tools/src/h5diff/h5diff_common.c @@ -56,6 +56,28 @@ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"vfd-info-2", require_arg, 'Z'}, {NULL, 0, '\0'}}; +static H5FD_onion_fapl_info_t onion_fa_g_1 = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + 32, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "first input file", /* comment */ +}; + +static H5FD_onion_fapl_info_t onion_fa_g_2 = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + 32, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "second input file", /* comment */ +}; + /*------------------------------------------------------------------------- * Function: check_options * @@ -451,7 +473,7 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const case '8': opts->vfd_info[0].type = VFD_BY_NAME; opts->vfd_info[0].u.name = H5_optarg; - opts->custom_vol[0] = TRUE; + opts->custom_vfd[0] = TRUE; break; case '9': @@ -476,6 +498,40 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const } } + /* If file 1 uses the onion VFD, get the revision number */ + if (opts->vfd_info[0].u.name && !HDstrcmp(opts->vfd_info[0].u.name, "onion")) { + if (opts->vfd_info[0].info) { + errno = 0; + onion_fa_g_1.revision_num = HDstrtoull(opts->vfd_info[0].info, NULL, 10); + if (errno == ERANGE) { + HDprintf("Invalid onion revision specified for file 1\n"); + usage(); + h5diff_exit(EXIT_FAILURE); + } + } + else + onion_fa_g_1.revision_num = 0; + + opts->vfd_info[0].info = &onion_fa_g_1; + } + + /* If file 2 uses the onion VFD, get the revision number */ + if (opts->vfd_info[1].u.name && !HDstrcmp(opts->vfd_info[1].u.name, "onion")) { + if (opts->vfd_info[1].info) { + errno = 0; + onion_fa_g_2.revision_num = HDstrtoull(opts->vfd_info[1].info, NULL, 10); + if (errno == ERANGE) { + HDprintf("Invalid onion revision specified for file 2\n"); + usage(); + h5diff_exit(EXIT_FAILURE); + } + } + else + onion_fa_g_2.revision_num = 0; + + opts->vfd_info[1].info = &onion_fa_g_2; + } + /* check options */ check_options(opts); diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index be68a47..3de1fbb 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -23,10 +23,13 @@ static hbool_t doxml_g = FALSE; static hbool_t useschema_g = TRUE; static const char *xml_dtd_uri_g = NULL; -static hbool_t use_custom_vol_g = FALSE; -static hbool_t use_custom_vfd_g = FALSE; -static h5tools_vol_info_t vol_info_g = {0}; -static h5tools_vfd_info_t vfd_info_g = {0}; +static hbool_t use_custom_vol_g = FALSE; +static hbool_t use_custom_vfd_g = FALSE; + +static h5tools_vol_info_t vol_info_g = {0}; +static h5tools_vfd_info_t vfd_info_g = {0}; + +static hbool_t get_onion_revision_count = FALSE; #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ @@ -51,6 +54,17 @@ static H5FD_hdfs_fapl_t hdfs_fa_g = { }; #endif /* H5_HAVE_LIBHDFS */ +static H5FD_onion_fapl_info_t onion_fa_g = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + 32, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "input file", /* comment */ +}; + /* module-scoped variables for XML option */ #define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd" #define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd" @@ -1305,6 +1319,29 @@ end_collect: } } + /* If the file uses the onion VFD, get the revision number */ + if (vfd_info_g.u.name && !HDstrcmp(vfd_info_g.u.name, "onion")) { + + if (vfd_info_g.info) { + if (!HDstrcmp(vfd_info_g.info, "revision_count")) + get_onion_revision_count = TRUE; + else { + errno = 0; + onion_fa_g.revision_num = HDstrtoull(vfd_info_g.info, NULL, 10); + if (errno == ERANGE) { + HDprintf("Invalid onion revision specified\n"); + goto error; + } + + HDprintf("Using revision %" PRIu64 "\n", onion_fa_g.revision_num); + } + } + else + onion_fa_g.revision_num = 0; + + vfd_info_g.info = &onion_fa_g; + } + parse_end: /* check for file name to be processed */ if (argc <= H5_optind) { @@ -1424,7 +1461,21 @@ main(int argc, char *argv[]) while (H5_optind < argc) { fname = HDstrdup(argv[H5_optind++]); - fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0); + /* A short cut to get the revision count of an onion file without opening the file */ + if (get_onion_revision_count && H5FD_ONION == H5Pget_driver(fapl_id)) { + uint64_t revision_count = 0; + + if (H5FDonion_get_revision_count(fname, fapl_id, &revision_count) < 0) { + error_msg("unable to create FAPL for file access\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + + HDprintf("The number of revisions for the onion file is %" PRIu64 "\n", revision_count); + goto done; + } + else + fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0); if (fid < 0) { error_msg("unable to open file \"%s\"\n", fname); diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index a805b3d..0d3bc9a 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -76,6 +76,17 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"dst-vfd-info", require_arg, 'Z'}, {NULL, 0, '\0'}}; +static H5FD_onion_fapl_info_t onion_fa_in_g = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + 32, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "input file", /* comment */ +}; + /*------------------------------------------------------------------------- * Function: usage * @@ -875,6 +886,23 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) ret_value = -1; } + /* If the input file uses the onion VFD, get the revision number */ + if (in_vfd_info.u.name && !HDstrcmp(in_vfd_info.u.name, "onion")) { + if (in_vfd_info.info) { + errno = 0; + onion_fa_in_g.revision_num = HDstrtoull(in_vfd_info.info, NULL, 10); + if (errno == ERANGE) { + HDprintf("Invalid onion revision specified for the input file\n"); + usage(h5tools_getprogname()); + exit(EXIT_FAILURE); + } + } + else + onion_fa_in_g.revision_num = 0; + + in_vfd_info.info = &onion_fa_in_g; + } + /* Setup FAPL for input and output file accesses */ if (custom_in_vol || custom_in_vfd) { if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL, diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake index 8e01605..21761c6 100644 --- a/tools/test/h5diff/CMakeTests.cmake +++ b/tools/test/h5diff/CMakeTests.cmake @@ -70,6 +70,13 @@ ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_strings2.h5 ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_eps1.h5 ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_eps2.h5 + # onion VFD files + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_objs.h5 + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_objs.h5.onion + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_ext.h5 + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_ext.h5.onion + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_1d.h5 + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_onion_dset_1d.h5.onion # tools/testfiles/vds ${HDF5_TOOLS_DIR}/testfiles/vds/1_a.h5 ${HDF5_TOOLS_DIR}/testfiles/vds/1_b.h5 @@ -295,6 +302,9 @@ ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_801.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_830.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_90.txt + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_900.txt + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_901.txt + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_902.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8625.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_8639.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_reg.txt @@ -932,6 +942,12 @@ h5diff_8639.out.err h5diff_90.out h5diff_90.out.err + h5diff_900.out + h5diff_900.out.err + h5diff_901.out + h5diff_901.out.err + h5diff_902.out + h5diff_902.out.err h5diff_v1.out h5diff_v1.out.err h5diff_v2.out @@ -1563,6 +1579,13 @@ ADD_H5_TEST (h5diff_v1 0 -v ${FILEV1} ${FILEV2}) ADD_H5_TEST (h5diff_v2 0 -r ${FILEV1} ${FILEV2}) ADD_H5_TEST (h5diff_v3 0 -c ${FILEV1} ${FILEV2}) +# ############################################################################## +# # onion VFD tests (serial only) +# ############################################################################## +ADD_SH5_TEST (h5diff_900 1 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_objs.h5 h5diff_onion_objs.h5) +ADD_SH5_TEST (h5diff_901 0 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_ext.h5 h5diff_onion_dset_ext.h5) +ADD_SH5_TEST (h5diff_902 1 -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_1d.h5 h5diff_onion_dset_1d.h5) + ############################################################################## ### P L U G I N T E S T S ############################################################################## diff --git a/tools/test/h5diff/h5diffgentest.c b/tools/test/h5diff/h5diffgentest.c index f388568..a66a7b8 100644 --- a/tools/test/h5diff/h5diffgentest.c +++ b/tools/test/h5diff/h5diffgentest.c @@ -38,30 +38,34 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024); *------------------------------------------------------------------------- */ -#define FILE1 "h5diff_basic1.h5" -#define FILE2 "h5diff_basic2.h5" -#define FILE3 "h5diff_types.h5" -#define FILE4 "h5diff_dtypes.h5" -#define FILE5 "h5diff_attr1.h5" -#define FILE6 "h5diff_attr2.h5" -#define FILE6a "h5diff_attr3.h5" -#define FILE7 "h5diff_dset1.h5" -#define FILE8 "h5diff_dset2.h5" -#define FILE8A "h5diff_dset3.h5" -#define FILE9 "h5diff_hyper1.h5" -#define FILE10 "h5diff_hyper2.h5" -#define FILE11 "h5diff_empty.h5" -#define FILE12 "h5diff_links.h5" -#define FILE13 "h5diff_softlinks.h5" -#define FILE14 "h5diff_linked_softlink.h5" -#define FILE15 "h5diff_extlink_src.h5" -#define FILE16 "h5diff_extlink_trg.h5" -#define FILE17 "h5diff_ext2softlink_src.h5" -#define FILE18 "h5diff_ext2softlink_trg.h5" -#define FILE19 "h5diff_dset_zero_dim_size1.h5" -#define FILE20 "h5diff_dset_zero_dim_size2.h5" -#define FILE21 "h5diff_dset_idx1.h5" -#define FILE22 "h5diff_dset_idx2.h5" +#define FILE1 "h5diff_basic1.h5" +#define FILE2 "h5diff_basic2.h5" +#define FILE3 "h5diff_types.h5" +#define FILE4 "h5diff_dtypes.h5" +#define FILE5 "h5diff_attr1.h5" +#define FILE6 "h5diff_attr2.h5" +#define FILE6a "h5diff_attr3.h5" +#define FILE7 "h5diff_dset1.h5" +#define FILE8 "h5diff_dset2.h5" +#define FILE8A "h5diff_dset3.h5" +#define FILE9 "h5diff_hyper1.h5" +#define FILE10 "h5diff_hyper2.h5" +#define FILE11 "h5diff_empty.h5" +#define FILE12 "h5diff_links.h5" +#define FILE13 "h5diff_softlinks.h5" +#define FILE14 "h5diff_linked_softlink.h5" +#define FILE15 "h5diff_extlink_src.h5" +#define FILE16 "h5diff_extlink_trg.h5" +#define FILE17 "h5diff_ext2softlink_src.h5" +#define FILE18 "h5diff_ext2softlink_trg.h5" +#define FILE19 "h5diff_dset_zero_dim_size1.h5" +#define FILE20 "h5diff_dset_zero_dim_size2.h5" +#define FILE21 "h5diff_dset_idx1.h5" +#define FILE22 "h5diff_dset_idx2.h5" +#define FILE23 "h5diff_onion_dset_1d.h5" +#define FILE24 "h5diff_onion_objs.h5" +#define FILE25 "h5diff_onion_dset_ext.h5" + #define DANGLE_LINK_FILE1 "h5diff_danglelinks1.h5" #define DANGLE_LINK_FILE2 "h5diff_danglelinks2.h5" #define GRP_RECURSE_FILE1 "h5diff_grp_recurse1.h5" @@ -111,6 +115,11 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024); #define SPACE1_DIM1 0 #define SPACE1_DIM2 0 +/* For Onion VFD */ +#define ONION_TEST_FIXNAME_SIZE 1024 +#define ONION_TEST_PAGE_SIZE (uint32_t)32 +#define ONE_DIM_SIZE 16 + /* Error macros */ #define AT() HDprintf("ERROR at %s:%d in %s()...\n", __FILE__, __LINE__, __func__); #define PROGRAM_ERROR \ @@ -179,6 +188,11 @@ static void test_objs_nocomparables(const char *fname1, const char *fname2); static void test_objs_strings(const char *fname, const char *fname2); static void test_double_epsilon(const char *fname1, const char *fname2); +/* Generate the files for testing Onion VFD */ +static int test_onion_1d_dset(const char *fname); +static int test_onion_create_delete_objects(const char *fname); +static int test_onion_dset_extension(const char *fname); + /* called by test_attributes() and test_datasets() */ static void write_attr_strings(hid_t loc_id, const char *dset_name, hid_t fid, int make_diffs); static void write_attr_in(hid_t loc_id, const char *dset_name, hid_t fid, int make_diffs); @@ -298,9 +312,676 @@ main(void) /* double dataset and epsilion. HDFFV-10897 */ test_double_epsilon(DIFF_EPS1, DIFF_EPS2); + /* Generate the files for testing Onion VFD */ + test_onion_1d_dset(FILE23); + test_onion_create_delete_objects(FILE24); + test_onion_dset_extension(FILE25); + return EXIT_SUCCESS; } +/* Structure to collect the onion filepaths in one place. */ +struct onion_filepaths { + char *canon; + char *onion; + char *recovery; +}; + +/* Allocate and populate filepaths with h5_fixname'd strings as appropriate. + * Should be released with onion_filepaths_destroy() when done. + */ +static struct onion_filepaths * +onion_filepaths_init(const char *basename) +{ + struct onion_filepaths *paths = NULL; + + if (NULL == (paths = HDcalloc(1, sizeof(struct onion_filepaths)))) + goto error; + + if (NULL == (paths->canon = HDstrdup(basename))) + goto error; + + if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE))) + goto error; + HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon); + + if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE))) + goto error; + HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon); + + return paths; + +error: + if (paths != NULL) { + HDfree(paths->canon); + HDfree(paths->onion); + HDfree(paths->recovery); + HDfree(paths); + } + return NULL; +} + +static void +onion_filepaths_destroy(struct onion_filepaths *s) +{ + if (s) { + HDfree(s->canon); + HDfree(s->onion); + HDfree(s->recovery); + HDfree(s); + } +} + +static int +test_onion_1d_dset(const char *fname) +{ + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[2] = {1, ONE_DIM_SIZE}, maxdims[2] = {1, ONE_DIM_SIZE}; + int wdata[1][ONE_DIM_SIZE], /* Write buffer */ + fillval; + + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hid_t file_id = H5I_INVALID_HID; + + /* Setup */ + onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS); + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + goto error; + + if ((paths = onion_filepaths_init(fname)) == NULL) + goto error; + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + /* Initialize data */ + for (int i = 0; i < ONE_DIM_SIZE; i++) + wdata[0][i] = i; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create dataspace with unlimited dimensions */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + goto error; + + /* Create the dataset creation property list */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + goto error; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + goto error; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]) < 0) + goto error; + + /* Close and release resources */ + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Dclose(dset) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + goto error; + + int dset_data[1][ONE_DIM_SIZE]; + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data[0][i] = i + ONE_DIM_SIZE; + + if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file_id) < 0) + goto error; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + goto error; + + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data[0][i] = i + 2048; + + if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + goto error; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + goto error; + file_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Third revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + goto error; + + for (int i = 0; i < ONE_DIM_SIZE; i += 20) + dset_data[0][i] = i + 3072; + + if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + goto error; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file_id) < 0) + goto error; + if (H5Pclose(fapl_id) < 0) + goto error; + if (H5Pclose(onion_info.backing_fapl_id) < 0) + goto error; + + onion_filepaths_destroy(paths); + + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(onion_info.backing_fapl_id); + H5Pclose(fapl_id); + H5Dclose(dset); + H5Sclose(space); + H5Fclose(file_id); + } + H5E_END_TRY; + + return -1; +} /* test_onion_1d_dset */ + +static int +test_onion_create_delete_objects(const char *fname) +{ + struct onion_filepaths *paths = NULL; + + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + + hid_t fapl_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t attr_space_id = H5I_INVALID_HID; + hid_t attr_id = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + + hsize_t attr_dim[1] = {4}; + hsize_t dims[2] = {4, 4}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk[2] = {4, 4}; + int wdata[4][4]; /* Write buffer */ + + int fillval; + + /* Set up */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + goto error; + + if ((paths = onion_filepaths_init(fname)) == NULL) + goto error; + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + + /* + * Initialize data. + */ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* + * Create a new file using the default properties. + */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* + * Create dataspace with unlimited dimensions. + */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + goto error; + + /* + * Create the dataset creation property list, and set the chunk + * size. + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + goto error; + + /* + * Set the fill value for the dataset. + */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + goto error; + + /* + * Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + goto error; + + /* + * Create the dataset using the dataset creation property list. + */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* + * Write the data to the dataset. + */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and add a dataset (DS2) to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* + * Create the dataset using the dataset creation property list. + */ + if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* + * Write the data to the dataset. + */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and remove the dataset (DS2), + * which was added during the first revision. + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0) + goto error; + + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Third revision: open the file with Onion VFD and add an attribute to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* Create dataspace for attribute */ + attr_space_id = H5Screate_simple(1, attr_dim, NULL); + + if ((attr_id = + H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + if (H5Sclose(attr_space_id) < 0) + goto error; + if (H5Aclose(attr_id) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Fourth revision: open the file with Onion VFD and delete the attribute + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if (H5Adelete(file, "file_attribute") < 0) + goto error; + + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Fifth revision: open the file with Onion VFD and add a group to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + if (H5Gclose(group_id) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Sixth revision: open the file with Onion VFD and delete the newly added group + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0) + goto error; + + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /* + * Close and release resources. + */ + if (H5Pclose(onion_info.backing_fapl_id) < 0) + goto error; + if (H5Pclose(fapl_id) < 0) + goto error; + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + + onion_filepaths_destroy(paths); + + return 0; +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + if (dset != H5I_INVALID_HID) + (void)H5Dclose(dset); + if (file != H5I_INVALID_HID) + (void)H5Fclose(file); + if (fapl_id != H5I_INVALID_HID) + (void)H5Pclose(fapl_id); + if (onion_info.backing_fapl_id != H5I_INVALID_HID) + H5Pclose(onion_info.backing_fapl_id); + + return -1; +} /* test_onion_create_delete_objects */ + +static int +test_onion_dset_extension(const char *fname) +{ + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset_space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[2] = {4, 4}, maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}, chunk[2] = {4, 4}; + hsize_t size[2], offset[2]; + int wdata[4][4], /* Write buffer */ + fillval; + + /* Setup */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + goto error; + + if ((paths = onion_filepaths_init(fname)) == NULL) + goto error; + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + /* Initialize data */ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create dataspace with unlimited dimensions */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + goto error; + + /* Create the dataset creation property list, and set the chunk + * size. + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + goto error; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + goto error; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + goto error; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and extend the dataset + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* Open the dataset */ + if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0) + goto error; + + /* Extend the dataset and double the rows */ + size[0] = 2 * dims[0]; + size[1] = dims[1]; + if (H5Dset_extent(dset, size) < 0) + goto error; + + if ((dset_space = H5Dget_space(dset)) < 0) + goto error; + + offset[0] = dims[0]; + offset[1] = 0; + if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0) + goto error; + + /* Write the data to the dataset. */ + if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Sclose(dset_space) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and shrink the dataset + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* Open the dataset */ + if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0) + goto error; + + /* Extend the dataset and shrink back the size */ + if (H5Dset_extent(dset, dims) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /* Close and release resources. */ + if (H5Pclose(onion_info.backing_fapl_id) < 0) + goto error; + if (H5Pclose(fapl_id) < 0) + goto error; + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + + onion_filepaths_destroy(paths); + + return 0; +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file); + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} /* test_onion_dset_extension */ + /*------------------------------------------------------------------------- * Function: test_basic * diff --git a/tools/test/h5diff/testfiles/h5diff_900.txt b/tools/test/h5diff/testfiles/h5diff_900.txt new file mode 100644 index 0000000..eca7872 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_900.txt @@ -0,0 +1,12 @@ + +file1 file2 +--------------------------------------- + x x / + x x /DS1 + x /DS2 + +group : </> and </> +0 differences found +dataset: </DS1> and </DS1> +0 differences found +EXIT CODE: 1 diff --git a/tools/test/h5diff/testfiles/h5diff_901.txt b/tools/test/h5diff/testfiles/h5diff_901.txt new file mode 100644 index 0000000..eb1f7d9 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_901.txt @@ -0,0 +1,17 @@ + +file1 file2 +--------------------------------------- + x x / + x x /DS1 + +group : </> and </> +0 differences found +dataset: </DS1> and </DS1> +Not comparable: </DS1> has rank 2, dimensions [4x4], max dimensions [18446744073709551615x18446744073709551615] +and </DS1> has rank 2, dimensions [8x4], max dimensions [18446744073709551615x18446744073709551615] +0 differences found +-------------------------------- +Some objects are not comparable +-------------------------------- +Use -c for a list of objects without details of differences. +EXIT CODE: 0 diff --git a/tools/test/h5diff/testfiles/h5diff_902.txt b/tools/test/h5diff/testfiles/h5diff_902.txt new file mode 100644 index 0000000..3863e16 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_902.txt @@ -0,0 +1,30 @@ + +file1 file2 +--------------------------------------- + x x / + x x /DS1 + +group : </> and </> +0 differences found +dataset: </DS1> and </DS1> +size: [1x16] [1x16] +position DS1 DS1 difference +------------------------------------------------------------ +[ 0 0 ] 0 16 16 +[ 0 1 ] 1 17 16 +[ 0 2 ] 2 18 16 +[ 0 3 ] 3 19 16 +[ 0 4 ] 4 20 16 +[ 0 5 ] 5 21 16 +[ 0 6 ] 6 22 16 +[ 0 7 ] 7 23 16 +[ 0 8 ] 8 24 16 +[ 0 9 ] 9 25 16 +[ 0 10 ] 10 26 16 +[ 0 11 ] 11 27 16 +[ 0 12 ] 12 28 16 +[ 0 13 ] 13 29 16 +[ 0 14 ] 14 30 16 +[ 0 15 ] 15 31 16 +16 differences found +EXIT CODE: 1 diff --git a/tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5 b/tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5 Binary files differnew file mode 100644 index 0000000..5e53fba --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5 diff --git a/tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5.onion b/tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5.onion Binary files differnew file mode 100644 index 0000000..95893f2 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_onion_dset_1d.h5.onion diff --git a/tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5 b/tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5 Binary files differnew file mode 100644 index 0000000..369ace6 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5 diff --git a/tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5.onion b/tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5.onion Binary files differnew file mode 100644 index 0000000..291f9b3 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_onion_dset_ext.h5.onion diff --git a/tools/test/h5diff/testfiles/h5diff_onion_objs.h5 b/tools/test/h5diff/testfiles/h5diff_onion_objs.h5 Binary files differnew file mode 100644 index 0000000..369ace6 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_onion_objs.h5 diff --git a/tools/test/h5diff/testfiles/h5diff_onion_objs.h5.onion b/tools/test/h5diff/testfiles/h5diff_onion_objs.h5.onion Binary files differnew file mode 100644 index 0000000..7811f83 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_onion_objs.h5.onion diff --git a/tools/test/h5diff/testh5diff.sh.in b/tools/test/h5diff/testh5diff.sh.in index 6871463..4d232a7 100644 --- a/tools/test/h5diff/testh5diff.sh.in +++ b/tools/test/h5diff/testh5diff.sh.in @@ -124,6 +124,12 @@ $SRC_H5DIFF_TESTFILES/h5diff_strings1.h5 $SRC_H5DIFF_TESTFILES/h5diff_strings2.h5 $SRC_H5DIFF_TESTFILES/h5diff_eps1.h5 $SRC_H5DIFF_TESTFILES/h5diff_eps2.h5 +$SRC_H5DIFF_TESTFILES/h5diff_onion_objs.h5 +$SRC_H5DIFF_TESTFILES/h5diff_onion_objs.h5.onion +$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_ext.h5 +$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_ext.h5.onion +$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_1d.h5 +$SRC_H5DIFF_TESTFILES/h5diff_onion_dset_1d.h5.onion $SRC_TOOLS_TESTFILES/tvlstr.h5 " @@ -356,6 +362,9 @@ $SRC_H5DIFF_TESTFILES/h5diff_800.txt $SRC_H5DIFF_TESTFILES/h5diff_801.txt $SRC_H5DIFF_TESTFILES/h5diff_830.txt $SRC_H5DIFF_TESTFILES/h5diff_90.txt +$SRC_H5DIFF_TESTFILES/h5diff_900.txt +$SRC_H5DIFF_TESTFILES/h5diff_901.txt +$SRC_H5DIFF_TESTFILES/h5diff_902.txt $SRC_H5DIFF_TESTFILES/h5diff_8625.txt $SRC_H5DIFF_TESTFILES/h5diff_8639.txt $SRC_H5DIFF_TESTFILES/h5diff_reg.txt @@ -1214,6 +1223,15 @@ TOOLTEST h5diff_v1.txt -v 1_vds.h5 2_vds.h5 TOOLTEST h5diff_v2.txt -r 1_vds.h5 2_vds.h5 TOOLTEST h5diff_v3.txt -c 1_vds.h5 2_vds.h5 +# ############################################################################## +# Onion VFD tests +# ############################################################################## +# These tests won't pass under ph5diff +if test -z "$pmode"; then + TOOLTEST h5diff_900.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_objs.h5 h5diff_onion_objs.h5 + TOOLTEST h5diff_901.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_ext.h5 h5diff_onion_dset_ext.h5 + TOOLTEST h5diff_902.txt -r -v --vfd-name-1 onion --vfd-info-1 0 --vfd-name-2 onion --vfd-info-2 1 h5diff_onion_dset_1d.h5 h5diff_onion_dset_1d.h5 +fi # ############################################################################## # # END diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index 933aba9..4fa9a8f 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -211,6 +211,11 @@ ${HDF5_TOOLS_DIR}/testfiles/trefer_paramR.ddl ${HDF5_TOOLS_DIR}/testfiles/trefer_reg_1dR.ddl ${HDF5_TOOLS_DIR}/testfiles/trefer_regR.ddl + # Onion VFD files + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.ddl + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.ddl + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.ddl + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_revision_count.ddl ) set (HDF5_N_REFERENCE_FILES tall-3 @@ -350,7 +355,13 @@ ${HDF5_TOOLS_DIR}/testfiles/trefer_param.h5 ${HDF5_TOOLS_DIR}/testfiles/trefer_reg_1d.h5 ${HDF5_TOOLS_DIR}/testfiles/trefer_reg.h5 - + # Onion VFD files + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5 + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5.onion + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5 + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5.onion + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5 + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5.onion ) set (HDF5_ERROR_REFERENCE_TEST_FILES ${PROJECT_SOURCE_DIR}/errfiles/filter_fail.err @@ -1187,6 +1198,13 @@ ADD_H5_TEST (tCVE_2018_11206_fill_old 1 tCVE_2018_11206_fill_old.h5) ADD_H5_TEST (tCVE_2018_11206_fill_new 1 tCVE_2018_11206_fill_new.h5) + # onion VFD tests + ADD_H5_TEST (tst_onion_objs 0 --enable-error-stack --vfd-name onion --vfd-info 3 tst_onion_objs.h5) + ADD_H5_TEST (tst_onion_dset_ext 0 --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_ext.h5) + ADD_H5_TEST (tst_onion_dset_1d 0 --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_1d.h5) + ADD_H5_TEST (tst_onion_revision_count 0 --enable-error-stack --vfd-name onion --vfd-info revision_count tst_onion_objs.h5) + + ############################################################################## ### P L U G I N T E S T S ############################################################################## diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 30e0c46..08f241c 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -117,6 +117,13 @@ #define FILE87 "tintsnodata.h5" #define FILE88 "tldouble_scalar.h5" #define FILE89 "tfloatsattrs.h5" +#define FILE90 "tst_onion_dset_1d.h5" +#define FILE91 "tst_onion_objs.h5" +#define FILE92 "tst_onion_dset_ext.h5" + +#define ONION_TEST_FIXNAME_SIZE 1024 +#define ONION_TEST_PAGE_SIZE (uint32_t)32 +#define ONE_DIM_SIZE 16 /*------------------------------------------------------------------------- * prototypes @@ -11350,6 +11357,671 @@ error: H5E_END_TRY; } /* gen_err_attr_dspace() */ +/* Structure to collect the onion filepaths in one place. */ +struct onion_filepaths { + char *canon; + char *onion; + char *recovery; +}; + +/* Allocate and populate filepaths with h5_fixname'd strings as appropriate. + * Should be released with onion_filepaths_destroy() when done. + */ +static struct onion_filepaths * +onion_filepaths_init(const char *basename) +{ + struct onion_filepaths *paths = NULL; + + if (NULL == (paths = HDcalloc(1, sizeof(struct onion_filepaths)))) + goto error; + + if (NULL == (paths->canon = HDstrdup(basename))) + goto error; + + if (NULL == (paths->onion = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE))) + goto error; + HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", paths->canon); + + if (NULL == (paths->recovery = HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE))) + goto error; + HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", paths->canon); + + return paths; + +error: + if (paths != NULL) { + HDfree(paths->canon); + HDfree(paths->onion); + HDfree(paths->recovery); + HDfree(paths); + } + return NULL; +} + +static void +onion_filepaths_destroy(struct onion_filepaths *s) +{ + if (s) { + HDfree(s->canon); + HDfree(s->onion); + HDfree(s->recovery); + HDfree(s); + } +} + +static int +gent_onion_1d_dset(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hsize_t dims[2] = {1, ONE_DIM_SIZE}; + hsize_t maxdims[2] = {1, ONE_DIM_SIZE}; + int wdata[1][ONE_DIM_SIZE]; + int fillval; + + /* Setup */ + onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS); + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + goto error; + + if ((paths = onion_filepaths_init(FILE90)) == NULL) + goto error; + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + /* Initialize data */ + for (int i = 0; i < ONE_DIM_SIZE; i++) + wdata[0][i] = i; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create dataspace with unlimited dimensions */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + goto error; + + /* Create the dataset creation property list */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + goto error; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + goto error; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]) < 0) + goto error; + + /* Close and release resources */ + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Dclose(dset) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + goto error; + + int dset_data[1][ONE_DIM_SIZE]; + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data[0][i] = i + ONE_DIM_SIZE; + + if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file_id) < 0) + goto error; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + goto error; + + for (int i = 0; i < ONE_DIM_SIZE; i++) + dset_data[0][i] = i + 2048; + + if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + goto error; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file_id) < 0) + goto error; + file_id = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Third revision: open the file with Onion VFD and change the data + *---------------------------------------------------------------------- + */ + if ((file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((dset = H5Dopen2(file_id, "DS1", H5P_DEFAULT)) < 0) + goto error; + + for (int i = 0; i < ONE_DIM_SIZE; i += 20) + dset_data[0][i] = i + 3072; + + if (H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data) < 0) + goto error; + + /* CLEANUP */ + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file_id) < 0) + goto error; + if (H5Pclose(fapl_id) < 0) + goto error; + if (H5Pclose(onion_info.backing_fapl_id) < 0) + goto error; + + onion_filepaths_destroy(paths); + + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(onion_info.backing_fapl_id); + H5Pclose(fapl_id); + H5Dclose(dset); + H5Sclose(space); + H5Fclose(file_id); + } + H5E_END_TRY; + + return -1; +} /* gent_onion_1d_dset */ + +static int +gent_onion_create_delete_objects(void) +{ + struct onion_filepaths *paths = NULL; + + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + + hid_t fapl_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t attr_space_id = H5I_INVALID_HID; + hid_t attr_id = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + + hsize_t attr_dim[1] = {4}; + hsize_t dims[2] = {4, 4}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk[2] = {4, 4}; + int wdata[4][4]; + + int fillval; + + /* Set up */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + goto error; + + if ((paths = onion_filepaths_init(FILE91)) == NULL) + goto error; + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + + /* + * Initialize data. + */ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* + * Create a new file using the default properties. + */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* + * Create dataspace with unlimited dimensions. + */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + goto error; + + /* + * Create the dataset creation property list, and set the chunk + * size. + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + goto error; + + /* + * Set the fill value for the dataset. + */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + goto error; + + /* + * Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + goto error; + + /* + * Create the dataset using the dataset creation property list. + */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* + * Write the data to the dataset. + */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and add a dataset (DS2) to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* + * Create the dataset using the dataset creation property list. + */ + if ((dset = H5Dcreate2(file, "DS2", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* + * Write the data to the dataset. + */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and remove the dataset (DS2), + * which was added during the first revision. + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if (H5Ldelete(file, "DS2", H5P_DEFAULT) < 0) + goto error; + + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Third revision: open the file with Onion VFD and add an attribute to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* Create dataspace for attribute */ + attr_space_id = H5Screate_simple(1, attr_dim, NULL); + + if ((attr_id = + H5Acreate2(file, "file_attribute", H5T_STD_I32LE, attr_space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + if (H5Sclose(attr_space_id) < 0) + goto error; + if (H5Aclose(attr_id) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Fourth revision: open the file with Onion VFD and delete the attribute + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if (H5Adelete(file, "file_attribute") < 0) + goto error; + + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Fifth revision: open the file with Onion VFD and add a group to the file + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if ((group_id = H5Gcreate2(file, "new_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + if (H5Gclose(group_id) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Sixth revision: open the file with Onion VFD and delete the newly added group + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + if (H5Ldelete(file, "new_group", H5P_DEFAULT) < 0) + goto error; + + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /* + * Close and release resources. + */ + if (H5Pclose(onion_info.backing_fapl_id) < 0) + goto error; + if (H5Pclose(fapl_id) < 0) + goto error; + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + + onion_filepaths_destroy(paths); + + return 0; +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + if (dset != H5I_INVALID_HID) + (void)H5Dclose(dset); + if (file != H5I_INVALID_HID) + (void)H5Fclose(file); + if (fapl_id != H5I_INVALID_HID) + (void)H5Pclose(fapl_id); + if (onion_info.backing_fapl_id != H5I_INVALID_HID) + H5Pclose(onion_info.backing_fapl_id); + + return -1; +} /* gent_onion_create_delete_objects */ + +static int +gent_onion_dset_extension(void) +{ + hid_t fapl_id = H5I_INVALID_HID; + hid_t file = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset_space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation flags, was H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT */ + "initial commit" /* comment */ + }; + hsize_t dims[2] = {4, 4}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk[2] = {4, 4}; + hsize_t size[2]; + hsize_t offset[2]; + int wdata[4][4]; /* Write buffer */ + int fillval; + + /* Setup */ + if ((onion_info.backing_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + goto error; + + if ((paths = onion_filepaths_init(FILE92)) == NULL) + goto error; + + /*---------------------------------------------------------------------- + * Create the skeleton file (create the file without Onion VFD) + *---------------------------------------------------------------------- + */ + /* Initialize data */ + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + wdata[i][j] = i + j; + + /* Create a new file using the default properties */ + if ((file = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create dataspace with unlimited dimensions */ + if ((space = H5Screate_simple(2, dims, maxdims)) < 0) + goto error; + + /* Create the dataset creation property list, and set the chunk + * size. + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if (H5Pset_chunk(dcpl, 2, chunk) < 0) + goto error; + + /* Set the fill value for the dataset */ + fillval = 99; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval) < 0) + goto error; + + /* Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + goto error; + + /* Create the dataset using the dataset creation property list */ + if ((dset = H5Dcreate2(file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Write the data to the dataset */ + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + if (H5Fclose(file) < 0) + goto error; + + /*---------------------------------------------------------------------- + * First revision: open the file with Onion VFD and extend the dataset + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* Open the dataset */ + if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0) + goto error; + + /* Extend the dataset and double the rows */ + size[0] = 2 * dims[0]; + size[1] = dims[1]; + if (H5Dset_extent(dset, size) < 0) + goto error; + + if ((dset_space = H5Dget_space(dset)) < 0) + goto error; + + offset[0] = dims[0]; + offset[1] = 0; + if (H5Sselect_hyperslab(dset_space, H5S_SELECT_SET, offset, NULL, dims, NULL) < 0) + goto error; + + /* Write the data to the dataset. */ + if (H5Dwrite(dset, H5T_NATIVE_INT, space, dset_space, H5P_DEFAULT, wdata) < 0) + goto error; + + if (H5Sclose(dset_space) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /*---------------------------------------------------------------------- + * Second revision: open the file with Onion VFD and shrink the dataset + *---------------------------------------------------------------------- + */ + if ((file = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id)) < 0) + goto error; + + /* Open the dataset */ + if ((dset = H5Dopen2(file, "DS1", H5P_DEFAULT)) < 0) + goto error; + + /* Extend the dataset and shrink back the size */ + if (H5Dset_extent(dset, dims) < 0) + goto error; + + if (H5Dclose(dset) < 0) + goto error; + dset = H5I_INVALID_HID; + if (H5Fclose(file) < 0) + goto error; + file = H5I_INVALID_HID; + + /* Close and release resources. */ + if (H5Pclose(onion_info.backing_fapl_id) < 0) + goto error; + if (H5Pclose(fapl_id) < 0) + goto error; + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + + onion_filepaths_destroy(paths); + + return 0; +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Fclose(file); + H5Pclose(fapl_id); + H5Pclose(onion_info.backing_fapl_id); + } + H5E_END_TRY; + + return -1; +} /* gent_onion_dset_extension */ + int main(void) { @@ -11449,5 +12121,10 @@ main(void) gent_err_attr_dspace(); + /* Generate the files for testing Onion VFD */ + gent_onion_1d_dset(); + gent_onion_create_delete_objects(); + gent_onion_dset_extension(); + return 0; } diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index f9f3734..ce7f643 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -184,6 +184,12 @@ $SRC_H5DUMP_TESTFILES/tvms.h5 $SRC_H5DUMP_TESTFILES/err_attr_dspace.h5 $SRC_H5DUMP_TESTFILES/tCVE_2018_11206_fill_old.h5 $SRC_H5DUMP_TESTFILES/tCVE_2018_11206_fill_new.h5 +$SRC_H5DUMP_TESTFILES/tst_onion_objs.h5 +$SRC_H5DUMP_TESTFILES/tst_onion_objs.h5.onion +$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5 +$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.h5.onion +$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5 +$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.h5.onion " LIST_OTHER_TEST_FILES=" @@ -372,6 +378,10 @@ $SRC_H5DUMP_TESTFILES/h5dump-help.txt $SRC_H5DUMP_TESTFILES/out3.h5import $SRC_H5DUMP_TESTFILES/tbinregR.exp $SRC_H5DUMP_TESTFILES/err_attr_dspace.ddl +$SRC_H5DUMP_TESTFILES/tst_onion_objs.ddl +$SRC_H5DUMP_TESTFILES/tst_onion_dset_ext.ddl +$SRC_H5DUMP_TESTFILES/tst_onion_dset_1d.ddl +$SRC_H5DUMP_TESTFILES/tst_onion_revision_count.ddl " LIST_ERROR_TEST_FILES=" @@ -1486,6 +1496,12 @@ TOOLTEST err_attr_dspace.ddl err_attr_dspace.h5 TOOLTEST_FAIL tCVE_2018_11206_fill_old.h5 TOOLTEST_FAIL tCVE_2018_11206_fill_new.h5 +# test Onion VFD +TOOLTEST tst_onion_objs.ddl --enable-error-stack --vfd-name onion --vfd-info 3 tst_onion_objs.h5 +TOOLTEST tst_onion_dset_ext.ddl --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_ext.h5 +TOOLTEST tst_onion_dset_1d.ddl --enable-error-stack --vfd-name onion --vfd-info 1 tst_onion_dset_1d.h5 +TOOLTEST tst_onion_revision_count.ddl --enable-error-stack --vfd-name onion --vfd-info revision_count tst_onion_objs.h5 + # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index 09648ff..163bd00 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -122,6 +122,13 @@ ${HDF5_TOOLS_DIR}/testfiles/vds/5_b.h5 ${HDF5_TOOLS_DIR}/testfiles/vds/5_c.h5 ${HDF5_TOOLS_DIR}/testfiles/vds/5_vds.h5 + # tools/testfiles onion VFD files + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5 + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5.onion + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5 + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5.onion + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5 + ${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5.onion ) set (LIST_OTHER_TEST_FILES @@ -446,6 +453,50 @@ endif () endmacro () + macro (ADD_H5_DMP_NO_OPT_TEST testname testtype resultcode resultfile) + if ("${testtype}" STREQUAL "SKIP") + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_DMP-${testname} + COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}" + ) + set_property(TEST H5REPACK_DMP-${testname} PROPERTY DISABLED) + endif () + else () + add_test ( + NAME H5REPACK_DMP-${testname}-clear-objects + COMMAND ${CMAKE_COMMAND} -E remove testfiles/out-${testname}.${resultfile} + ) + set_tests_properties (H5REPACK_DMP-${testname}-clear-objects PROPERTIES + FIXTURES_REQUIRED clear_h5repack + ) + add_test ( + NAME H5REPACK_DMP-${testname} + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5repack${tgt_file_ext}> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) + set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES + DEPENDS H5REPACK_DMP-${testname}-clear-objects + ) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_DMP-h5dump-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5dump${tgt_file_ext}>" + -D "TEST_ARGS:STRING=out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}-${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${testname}.${resultfile}.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5REPACK_DMP-h5dump-${testname} PROPERTIES + DEPENDS "H5REPACK_DMP-${testname}" + ) + endif () + endif () + endmacro () + macro (ADD_H5_DIFF_TEST testname testtype resultcode testfile) if ("${testtype}" STREQUAL "SKIP") if (NOT HDF5_ENABLE_USING_MEMCHECKER) @@ -1665,6 +1716,10 @@ ADD_H5_DMP_TEST (textlink-mergeprune "TEST" 0 textlink.h5 --merge --prune --enab ### HDFFV-11128 needs fixed to enable the following test #ADD_H5_DMP_TEST (textlinktar-mergeprune "TEST" 0 textlinktar.h5 --merge --prune --enable-error-stack) +ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_1d "TEST" 0 tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1) +ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_ext "TEST" 0 tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1) +ADD_H5_DMP_NO_OPT_TEST (tst_onion_objs "TEST" 0 tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1) + ############################################################################## ### P L U G I N T E S T S ############################################################################## diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index a241ea8..63d62bc 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -177,6 +177,13 @@ $SRC_TOOLS_TESTFILES/vds/5_a.h5 $SRC_TOOLS_TESTFILES/vds/5_b.h5 $SRC_TOOLS_TESTFILES/vds/5_c.h5 $SRC_TOOLS_TESTFILES/vds/5_vds.h5 +########tools/testfiles######## +$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5 +$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5.onion +$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5 +$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5.onion +$SRC_TOOLS_TESTFILES/tst_onion_objs.h5 +$SRC_TOOLS_TESTFILES/tst_onion_objs.h5.onion " LIST_OTHER_TEST_FILES=" @@ -251,6 +258,10 @@ $SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst $SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst $SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst $SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst +########onion#files######## +$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_1d.h5.ddl +$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_ext.h5.ddl +$SRC_H5REPACK_TESTFILES/onion.tst_onion_objs.h5.ddl " # @@ -1130,6 +1141,56 @@ TOOLTEST_DUMP() rm -f $outfile } +# This is same as TOOLTEST_DUMP() with comparing h5dump output +# without any option +# +TOOLTEST_DUMP_NO_OPT() +{ + infile=$2 + outfile=out-$1.$2 + expect="$TESTDIR/$1.$2.ddl" + actual="$TESTDIR/out-$1.$2.out" + actual_err="$TESTDIR/out-$1.$2.err" + + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5dump output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DUMP_BIN $outfile + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + RET=$? + + fi + + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + # This is similar to TOOLTEST_DUMP(). # Test h5repack with options added for paged aggregation. # h5stat is used on the repacked file and the expected output @@ -1830,6 +1891,9 @@ TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-sta #TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack ### HDFFV-11128 needs fixed to enable the following test #TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack +TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1 +TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1 +TOOLTEST_DUMP_NO_OPT onion tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1 # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl new file mode 100644 index 0000000..06ecf1a --- /dev/null +++ b/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl @@ -0,0 +1,11 @@ +HDF5 "out-onion.tst_onion_dset_1d.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 1, 16 ) / ( 1, 16 ) } + DATA { + (0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + } + } +} +} diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl new file mode 100644 index 0000000..39c1900 --- /dev/null +++ b/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl @@ -0,0 +1,18 @@ +HDF5 "out-onion.tst_onion_dset_ext.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6, + (4,0): 0, 1, 2, 3, + (5,0): 1, 2, 3, 4, + (6,0): 2, 3, 4, 5, + (7,0): 3, 4, 5, 6 + } + } +} +} diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl new file mode 100644 index 0000000..fa4ab4f --- /dev/null +++ b/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl @@ -0,0 +1,24 @@ +HDF5 "out-onion.tst_onion_objs.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6 + } + } + DATASET "DS2" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6 + } + } +} +} diff --git a/tools/testfiles/tst_onion_dset_1d.ddl b/tools/testfiles/tst_onion_dset_1d.ddl new file mode 100644 index 0000000..adca75f --- /dev/null +++ b/tools/testfiles/tst_onion_dset_1d.ddl @@ -0,0 +1,12 @@ +Using revision 1 +HDF5 "tst_onion_dset_1d.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 1, 16 ) / ( 1, 16 ) } + DATA { + (0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + } + } +} +} diff --git a/tools/testfiles/tst_onion_dset_1d.h5 b/tools/testfiles/tst_onion_dset_1d.h5 Binary files differnew file mode 100644 index 0000000..5042ebf --- /dev/null +++ b/tools/testfiles/tst_onion_dset_1d.h5 diff --git a/tools/testfiles/tst_onion_dset_1d.h5.onion b/tools/testfiles/tst_onion_dset_1d.h5.onion Binary files differnew file mode 100644 index 0000000..a86670e --- /dev/null +++ b/tools/testfiles/tst_onion_dset_1d.h5.onion diff --git a/tools/testfiles/tst_onion_dset_ext.ddl b/tools/testfiles/tst_onion_dset_ext.ddl new file mode 100644 index 0000000..2405774 --- /dev/null +++ b/tools/testfiles/tst_onion_dset_ext.ddl @@ -0,0 +1,19 @@ +Using revision 1 +HDF5 "tst_onion_dset_ext.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6, + (4,0): 0, 1, 2, 3, + (5,0): 1, 2, 3, 4, + (6,0): 2, 3, 4, 5, + (7,0): 3, 4, 5, 6 + } + } +} +} diff --git a/tools/testfiles/tst_onion_dset_ext.h5 b/tools/testfiles/tst_onion_dset_ext.h5 Binary files differnew file mode 100644 index 0000000..479aaa8 --- /dev/null +++ b/tools/testfiles/tst_onion_dset_ext.h5 diff --git a/tools/testfiles/tst_onion_dset_ext.h5.onion b/tools/testfiles/tst_onion_dset_ext.h5.onion Binary files differnew file mode 100644 index 0000000..b5cbdb4 --- /dev/null +++ b/tools/testfiles/tst_onion_dset_ext.h5.onion diff --git a/tools/testfiles/tst_onion_objs.ddl b/tools/testfiles/tst_onion_objs.ddl new file mode 100644 index 0000000..c66275b --- /dev/null +++ b/tools/testfiles/tst_onion_objs.ddl @@ -0,0 +1,22 @@ +Using revision 3 +HDF5 "tst_onion_objs.h5" { +GROUP "/" { + ATTRIBUTE "file_attribute" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 4 ) / ( 4 ) } + DATA { + (0): 0, 0, 0, 0 + } + } + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6 + } + } +} +} diff --git a/tools/testfiles/tst_onion_objs.h5 b/tools/testfiles/tst_onion_objs.h5 Binary files differnew file mode 100644 index 0000000..479aaa8 --- /dev/null +++ b/tools/testfiles/tst_onion_objs.h5 diff --git a/tools/testfiles/tst_onion_objs.h5.onion b/tools/testfiles/tst_onion_objs.h5.onion Binary files differnew file mode 100644 index 0000000..4abc174 --- /dev/null +++ b/tools/testfiles/tst_onion_objs.h5.onion diff --git a/tools/testfiles/tst_onion_revision_count.ddl b/tools/testfiles/tst_onion_revision_count.ddl new file mode 100644 index 0000000..4f2e1db --- /dev/null +++ b/tools/testfiles/tst_onion_revision_count.ddl @@ -0,0 +1 @@ +The number of revisions for the onion file is 6 |