/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * * the COPYING file, which can be found at the root of the source code * * distribution tree, or in https://www.hdfgroup.org/licenses. * * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hdf5.h" #include "h5test.h" /* The HDF5 test files */ static const char *FILENAME[] = { "h5clear_sec2_v3.h5", /* 0 -- sec2 file with superblock version 3 */ "h5clear_log_v3.h5", /* 1 -- log file with superblock version 3 */ "h5clear_sec2_v0.h5", /* 2 -- sec2 file with superblock version 0 */ "h5clear_sec2_v2.h5" /* 3 -- sec2 file with superblock version 2 */ }; const char *FILENAME_ENHANCE[] = { "h5clear_fsm_persist_equal.h5", /* 0: persisting free-space, stored EOA = actual EOF */ "h5clear_fsm_persist_greater.h5", /* 1: persisting free-space, stored EOA > actual EOF */ "h5clear_fsm_persist_less.h5", /* 2: persisting free-space, stored EOA < actual EOF */ "h5clear_fsm_persist_user_equal.h5", /* 3: user block, persisting free-space, stored EOA = actual EOF */ "h5clear_fsm_persist_user_greater.h5", /* 4: user block, persisting free-space, stored EOA > actual EOF */ "h5clear_fsm_persist_user_less.h5", /* 5: user block, persisting free-space, stored EOA < actual EOF */ "h5clear_status_noclose.h5", /* 6 -- v3 superblock, nonzero status_flags, no flush, exit, stored EOA < actual EOF */ "h5clear_fsm_persist_noclose.h5" /* 7 -- persisting free-space, no flush, exit, stored EOA < actual EOF */ }; #define KB 1024U #define CACHE_IMAGE_FILE "h5clear_mdc_image.h5" #define DSET "DSET" #define DATASET "dset" #define NUM_ELMTS 100 #define USERBLOCK 512 /*------------------------------------------------------------------------- * Function: gen_cache_image_file * * Purpose: To create a file with cache image feature enabled. * * Return: Success: 0 * Failure: 1 * * Programmer: Vailin Choi; March 2017 * *------------------------------------------------------------------------- */ static int gen_cache_image_file(const char *fname) { hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t did = -1, sid = H5I_INVALID_HID; /* Dataset ID, dataspace ID */ hid_t fapl = H5I_INVALID_HID; /* File access property list */ hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ hsize_t dims[2]; /* Dimension sizes */ hsize_t chunks[2]; /* Chunked dimension sizes */ int i, j; /* Local index variables */ struct { int arr[50][100]; } * buf; /* Buffer for data to write */ H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */ {H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, TRUE, FALSE, H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; /* Create and fill array */ buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; for (i = 0; i < 50; i++) for (j = 0; j < 100; j++) buf->arr[i][j] = i * j; /* Create a copy of file access property list */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; /* Enable latest format in fapl */ if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) goto error; /* Enable metadata cache image in fapl */ if (H5Pset_mdc_image_config(fapl, &cache_image_config) < 0) goto error; /* Create the file */ if ((fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) goto error; /* Create dataspace */ dims[0] = 50; dims[1] = 100; if ((sid = H5Screate_simple(2, dims, NULL)) < 0) goto error; /* Set up to create a chunked dataset */ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error; chunks[0] = 5; chunks[1] = 10; if (H5Pset_chunk(dcpl, 2, chunks) < 0) goto error; if ((did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; /* Write to the dataset */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) goto error; /* Closing */ if (H5Dclose(did) < 0) goto error; if (H5Pclose(dcpl) < 0) goto error; if (H5Pclose(fapl) < 0) goto error; if (H5Sclose(sid) < 0) goto error; if (H5Fclose(fid) < 0) goto error; free(buf); return 0; error: H5E_BEGIN_TRY { H5Pclose(dcpl); H5Sclose(sid); H5Dclose(did); H5Fclose(fid); H5Pclose(fapl); H5Pclose(dcpl); } H5E_END_TRY free(buf); return 1; } /* gen_cache_image_file() */ /*------------------------------------------------------------------------- * Function: gen_enhance_files * * Purpose: To create the first 6 files in FILENAME_ENHANCE[]: * (0) FILENAME_ENHANCE[0]: "h5clear_fsm_persist_equal.h5" * (1) FILENAME_ENHANCE[1]: "h5clear_fsm_persist_greater.h5" * (2) FILENAME_ENHANCE[2]: "h5clear_fsm_persist_less.h5" * (3) FILENAME_ENHANCE[3]: "h5clear_user_fsm_persist_equal.h5" * (4) FILENAME_ENHANCE[4]: "h5clear_user_fsm_persist_greater.h5" * (5) FILENAME_ENHANCE[5]: "h5clear_user_fsm_persist_less.h5" * After creating the files for #1, #2, #4 #5, write invalid EOA * value to the location where the EOA is stored in the superblock. * Also modify the chksum in the superblock due to this change. * * The first call to this routine (without user block) will generate * the first 3 files. * The second call to this routine (with user block) will generate * the last 3 files. * * Return: Success: 0 * Failure: 1 * * Programmer: Vailin Choi; March 2017 * *------------------------------------------------------------------------- */ static int gen_enhance_files(hbool_t user) { hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t fcpl = H5I_INVALID_HID; /* File creation property list */ hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ hid_t did = H5I_INVALID_HID; /* Dataset ID */ hsize_t dim[1]; /* Dimension sizes */ int data[NUM_ELMTS]; /* Buffer for data */ int fd = H5I_INVALID_HID; /* The file descriptor ID */ int64_t eoa; /* The EOA value */ uint32_t chksum; /* The chksum value */ int i = 0, j = 0, u = 0; /* Local index variable */ /* Get a copy of the default file creation property */ if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) goto error; /* Check to see if user block will be added */ if (user) { if (H5Pset_userblock(fcpl, (hsize_t)USERBLOCK) < 0) goto error; u = 3; } /* Set file space strategy and persisting free-space */ if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) goto error; /* * Create the file, then write invalid EOA to the file. */ for (i = 0 + u; i < 3 + u; i++) { /* Create the file with the file space strategy and persisting free-space */ if ((fid = H5Fcreate(FILENAME_ENHANCE[i], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) goto error; /* Create the dataset */ dim[0] = NUM_ELMTS; if ((sid = H5Screate_simple(1, dim, NULL)) < 0) goto error; if ((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; for (j = 0; j < NUM_ELMTS; j++) data[j] = j; /* Write the dataset */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) goto error; /* Closing */ if (H5Dclose(did) < 0) goto error; if (H5Sclose(sid) < 0) goto error; if (H5Fclose(fid) < 0) goto error; /* * No further action for: * --FILENAME_ENHANCE[0]: "h5clear_fsm_persist_equal.h5" * --FILENAME_ENHANCE[3]: "h5clear_fsm_persist_user_equal.h5", */ if (!(i % 3)) continue; /* * For the following files: * --FILENAME_ENHANCE[1]: "h5clear_fsm_persist_greater.h5" * --FILENAME_ENHANCE[2]: "h5clear_fsm_persist_less.h5" * --FILENAME_ENHANCE[4]: "h5clear_fsm_persist_greater.h5" * --FILENAME_ENHANCE[5]: "h5clear_fsm_persist_less.h5" * * Write invalid value to the location for stored eoa and * update the chksum value. */ /* Open the file */ if ((fd = open(FILENAME_ENHANCE[i], O_RDWR, 0663)) < 0) goto error; switch (i) { case 1: /* stored EOA is > EOF */ eoa = 3048; chksum = 268376587; break; case 2: /* stored EOA is < EOF */ eoa = 512; chksum = 372920305; break; case 4: /* with userblock, stored EOA > EOF */ eoa = 4000; chksum = 4168810027; break; case 5: /* with userblock, stored EOA < EOF */ eoa = 3000; chksum = 3716054346; break; default: break; } /* location of "end of file address" */ if (lseek(fd, (off_t)(28 + (user ? USERBLOCK : 0)), SEEK_SET) < 0) goto error; /* Write the bad eoa value to the file */ if (write(fd, &eoa, sizeof(eoa)) < 0) goto error; /* location of "superblock checksum" */ if (lseek(fd, (off_t)(44 + (user ? USERBLOCK : 0)), SEEK_SET) < 0) goto error; /* Write the chksum value to the file */ if (write(fd, &chksum, sizeof(chksum)) < 0) goto error; /* Close the file */ if (close(fd) < 0) goto error; } /* end for */ /* Close the property list */ if (H5Pclose(fcpl) < 0) goto error; return 0; error: H5E_BEGIN_TRY { H5Sclose(sid); H5Dclose(did); H5Fclose(fid); H5Pclose(fcpl); } H5E_END_TRY return 1; } /* gen_enhance_files() */ /*------------------------------------------------------------------------- * Function: main * * Purpose: Generate test files used by h5clear. * * (A) gen_cache_image_file(): * --generate a file with cache image feature * --"h5clear_mdc_image.h5" * (B) gen_enhance_files(): * --generate the first 6 files in FILENAME_ENHANCE[]: * (0) "h5clear_fsm_persist_equal.h5" * (1) "h5clear_fsm_persist_greater.h5" * (2) "h5clear_fsm_persist_less.h5" * (3) "h5clear_fsm_persist_user_equal.h5" * (4) "h5clear_fsm_persist_user_greater.h5" * (5) "h5clear_fsm_persist_user_less.h5" * * (C) Generate the following FILENAME[] files in main(): * (0a) "h5clear_sec2_v3.h5" * (0b) "latest_h5clear_sec2_v3.h5" * (1a) "h5clear_log_v3.h5", * (1b) "latest_h5clear_log_v3.h5" * (2) "h5clear_sec2_v0.h5" * (3) "h5clear_sec2_v2.h5" * * These HDF5 files are created with non-zero status_flags in * the superblock via flushing and exiting without closing the * library. * Due to file locking, status_flags in the superblock will be * nonzero after H5Fcreate. The library will clear status_flags * on file closing. * This program, after "H5Fcreate" the files, exits without * going through library closing. Thus, status_flags for these * files are not cleared. * The library will check consistency of status_flags when * opening a file with superblock >= v3 and will return error * accordingly. * The library will not check status_flags when opening a file * with < v3 superblock. * These files are used by "h5clear" to see if the tool clears * status_flags properly so users can open the files afterwards. * * (D) Generate the last two files in FILENAME_ENHANCE[] in main(): * (6) "h5clear_status_noclose.h5", * (7) "h5clear_fsm_persist_noclose.h5" * * Return: Success: 0 * Failure: 1 * * Programmer: Vailin Choi; July 2013 * *------------------------------------------------------------------------- */ int main(void) { hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t fcpl = H5I_INVALID_HID; /* File creation property list */ hid_t fapl = -1, new_fapl = H5I_INVALID_HID; /* File access property lists */ char fname[512]; /* File name */ unsigned new_format; /* To use latest library format or not */ hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ hid_t did = H5I_INVALID_HID; /* Dataset ID */ hsize_t dim[1]; /* Dimension sizes */ int data[NUM_ELMTS]; /* Buffer for data */ int i; /* Local index variables */ /* Generate a file with cache image feature enabled */ if (gen_cache_image_file(CACHE_IMAGE_FILE) < 0) goto error; /* Generate the first 6 files in FILENAME_ENHANCE[] */ if (gen_enhance_files(FALSE) < 0) goto error; if (gen_enhance_files(TRUE) < 0) goto error; /* * Generate files in FILENAME[] */ /* Create a copy of the file access property list */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; /* Copy the file access property list */ if ((new_fapl = H5Pcopy(fapl)) < 0) goto error; /* Set to latest library format */ if (H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) goto error; /* * Files created within this for loop will have v3 superblock and nonzero status_flags * --FILENAME[0]: "h5clear_sec2_v3.h5", "latest_h5clear_sec2_v3.h5" * --FILENAME[1]: "h5clear_log_v3.h5", "latest_h5clear_log_v3.h5" */ for (new_format = FALSE; new_format <= TRUE; new_format++) { hid_t fapl2, my_fapl; /* File access property lists */ /* Set to use the appropriate file access property list */ if (new_format) fapl2 = new_fapl; else fapl2 = fapl; /* * Create a sec2 file */ if ((my_fapl = H5Pcopy(fapl2)) < 0) goto error; /* Create the file */ HDsnprintf(fname, sizeof(fname), "%s%s", new_format ? "latest_" : "", FILENAME[0]); if ((fid = H5Fcreate(fname, H5F_ACC_TRUNC | (new_format ? 0 : H5F_ACC_SWMR_WRITE), H5P_DEFAULT, my_fapl)) < 0) goto error; /* Flush the file */ if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) goto error; /* Close the property list */ if (H5Pclose(my_fapl) < 0) goto error; /* * Create a log file */ /* Create a copy of file access property list */ if ((my_fapl = H5Pcopy(fapl2)) < 0) goto error; /* Setup the fapl for the log driver */ if (H5Pset_fapl_log(my_fapl, "append.log", (unsigned long long)H5FD_LOG_ALL, (size_t)(4 * KB)) < 0) goto error; /* Create the file */ HDsnprintf(fname, sizeof(fname), "%s%s", new_format ? "latest_" : "", FILENAME[1]); if ((fid = H5Fcreate(fname, H5F_ACC_TRUNC | (new_format ? 0 : H5F_ACC_SWMR_WRITE), H5P_DEFAULT, my_fapl)) < 0) goto error; /* Flush the file */ if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) goto error; /* Close the property list */ if (H5Pclose(my_fapl) < 0) goto error; } /* end for */ /* * Create a sec2 file with v0 superblock but nonzero status_flags: * FILENAME[2]: "h5clear_sec2_v0.h5" */ if ((fid = H5Fcreate(FILENAME[2], H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) goto error; /* Flush the file */ if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) goto error; /* * Create a sec2 file with v2 superblock but nonzero status_flags: * FILENAME[3]: "h5clear_sec2_v2.h5" */ if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) goto error; if (H5Pset_shared_mesg_nindexes(fcpl, 1) < 0) goto error; if (H5Pset_shared_mesg_index(fcpl, 0, H5O_SHMESG_DTYPE_FLAG, 50) < 0) goto error; if ((fid = H5Fcreate(FILENAME[3], H5F_ACC_TRUNC, fcpl, fapl)) < 0) goto error; /* Flush the file */ if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) goto error; /* Close the property lists */ if (H5Pclose(fapl) < 0) goto error; if (H5Pclose(new_fapl) < 0) goto error; if (H5Pclose(fcpl) < 0) goto error; /* * Create the last two files in FILENAME_ENHANCE[]: * --FILENAME_ENHANCE[6]: h5clear_status_noclose.h5 * --FILENAME_ENHANCE[7]: h5clear_fsm_persist_noclose.h5 */ /* * FILENAME_ENHANCE[6]: h5clear_status_noclose.h5 * --stored EOA < actual EOF * --version 3 superblock * --nonzero status_flags * --does not persist free-space * --does not flush the file, just exit without closing file: * --this file is similar to the user-suppplied test file attached with HDFFV-10347 */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; /* Set to latest format */ if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) goto error; /* Create file with SWMR-write access */ if ((fid = H5Fcreate(FILENAME_ENHANCE[6], H5F_ACC_TRUNC | H5F_ACC_SWMR_WRITE, H5P_DEFAULT, fapl)) < 0) goto error; /* Create the dataset */ dim[0] = NUM_ELMTS; if ((sid = H5Screate_simple(1, dim, NULL)) < 0) goto error; if ((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; for (i = 0; i < NUM_ELMTS; i++) data[i] = i; /* Write the dataset */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) goto error; /* Closing */ if (H5Dclose(did) < 0) goto error; if (H5Sclose(sid) < 0) goto error; if (H5Pclose(fapl) < 0) goto error; /* Does not flush and does not close the file */ /* * FILENAME_ENHANCE[7]: h5clear_fsm_persist_noclose.h5 * --stored EOA < actual EOF * --persisting free-space * --undefined fsinfo.eoa_pre_fsm_fsalloc * --undefined fsinfo.fs_addr * --does not flush the file, just exit without closing */ if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) goto error; /* Set file space strategy and persisting free-space */ if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) goto error; /* Create the file with the set file space info */ if ((fid = H5Fcreate(FILENAME_ENHANCE[7], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) goto error; /* Create the dataset */ dim[0] = NUM_ELMTS; if ((sid = H5Screate_simple(1, dim, NULL)) < 0) goto error; if ((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; for (i = 0; i < NUM_ELMTS; i++) data[i] = i; /* Write the dataset */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) goto error; /* Closing */ if (H5Dclose(did) < 0) goto error; if (H5Sclose(sid) < 0) goto error; if (H5Pclose(fcpl) < 0) goto error; /* Does not flush and does not close the file */ fflush(stdout); fflush(stderr); /* Not going through library closing by calling _exit(0) with success */ HD_exit(0); error: /* Exit with failure */ HD_exit(1); }