diff options
| -rw-r--r-- | MANIFEST | 2 | ||||
| -rw-r--r-- | hl/src/H5DO.c | 192 | ||||
| -rw-r--r-- | hl/src/H5DOpublic.h | 3 | ||||
| -rw-r--r-- | hl/test/Makefile.am | 5 | ||||
| -rw-r--r-- | hl/test/test_dset_append.c | 1196 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/H5AC.c | 132 | ||||
| -rw-r--r-- | src/H5ACprivate.h | 11 | ||||
| -rw-r--r-- | src/H5C.c | 297 | ||||
| -rw-r--r-- | src/H5Cprivate.h | 35 | ||||
| -rw-r--r-- | src/H5D.c | 38 | ||||
| -rw-r--r-- | src/H5Dint.c | 111 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 1 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 9 | ||||
| -rw-r--r-- | src/H5Dpublic.h | 4 | ||||
| -rw-r--r-- | src/H5Fint.c | 40 | ||||
| -rw-r--r-- | src/H5Fio.c | 45 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 3 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 13 | ||||
| -rw-r--r-- | src/H5Fpublic.h | 3 | ||||
| -rw-r--r-- | src/H5Oflush.c | 132 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 3 | ||||
| -rw-r--r-- | src/H5Pdapl.c | 131 | ||||
| -rw-r--r-- | src/H5Pdxpl.c | 7 | ||||
| -rw-r--r-- | src/H5Pfapl.c | 95 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 6 | ||||
| -rw-r--r-- | src/Makefile.am | 2 |
27 files changed, 2342 insertions, 175 deletions
@@ -722,6 +722,7 @@ ./src/H5Odtype.c ./src/H5Oefl.c ./src/H5Ofill.c +./src/H5Oflush.c ./src/H5Ofsinfo.c ./src/H5Oginfo.c ./src/H5Olayout.c @@ -2386,6 +2387,7 @@ ./hl/test/test_ds.c ./hl/test/test_ds_be.h5 ./hl/test/test_ds_le.h5 +./hl/test/test_dset_append.c ./hl/test/test_dset_opt.c ./hl/test/test_file_image.c ./hl/test/test_image.c diff --git a/hl/src/H5DO.c b/hl/src/H5DO.c index 99dbd93..bfadfaa 100644 --- a/hl/src/H5DO.c +++ b/hl/src/H5DO.c @@ -97,3 +97,195 @@ done: return(ret_value); } /* end H5DOwrite_chunk() */ + +/* + * Function: H5DOappend() + * + * Purpose: To append elements to a dataset. + * axis: the dataset dimension (zero-based) for the append + * extension: the # of elements to append for the axis-th dimension + * memtype: the datatype + * buf: buffer with data for the append + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Jan 2014 + * + * Note: + * This routine is copied from the fast forward feature branch: features/hdf5_ff + * src/H5FF.c:H5DOappend() with the following modifications: + * 1) Remove and replace macro calls such as + * FUNC_ENTER_API, H5TRACE, HGOTO_ERROR + * accordingly because hl does not have these macros + * 2) Replace H5I_get_type() by H5Iget_type() + * 3) Replace H5P_isa_class() by H5Pisa_class() + * 4) Fix a bug in the following: replace extension by size[axis] + * if(extension < old_size) { + * ret_value = FAIL; + * goto done; + * } + */ +herr_t +H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, + hid_t memtype, const void *buf) +{ + hbool_t created_dxpl = FALSE; /* Whether we created a DXPL */ + hsize_t size[H5S_MAX_RANK]; /* The new size (after extension */ + hsize_t old_size = 0; /* The size of the dimension to be extended */ + int sndims; /* Number of dimensions in dataspace (signed) */ + unsigned ndims; /* Number of dimensions in dataspace */ + hid_t space_id = FAIL; /* Old file space */ + hid_t new_space_id = FAIL; /* New file space (after extension) */ + hid_t mem_space_id = FAIL; /* Memory space for data buffer */ + hssize_t snelmts; /* Number of elements in selection (signed) */ + hsize_t nelmts; /* Number of elements in selection */ + hid_t dapl = FAIL; /* Dataset access property list */ + + hsize_t start[H5S_MAX_RANK]; /* H5Sselect_Hyperslab: starting offset */ + hsize_t count[H5S_MAX_RANK]; /* H5Sselect_hyperslab: # of blocks to select */ + hsize_t stride[H5S_MAX_RANK]; /* H5Sselect_hyperslab: # of elements to move when selecting */ + hsize_t block[H5S_MAX_RANK]; /* H5Sselect_hyperslab: # of elements in a block */ + + hsize_t *boundary = NULL; /* Boundary set in append flush property */ + H5D_append_cb_t append_cb; /* Callback function set in append flush property */ + void *udata; /* User data set in append flush property */ + hbool_t hit = FALSE; /* Boundary is hit or not */ + hsize_t k; /* Local index variable */ + unsigned u; /* Local index variable */ + herr_t ret_value = FAIL; /* Return value */ + + /* check arguments */ + if(H5I_DATASET != H5Iget_type(dset_id)) + goto done; + + /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ + if(H5P_DEFAULT == dxpl_id) { + if((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto done; + created_dxpl = TRUE; + } /* end if */ + else if(TRUE != H5Pisa_class(dxpl_id, H5P_DATASET_XFER)) + goto done; + + /* Get the dataspace of the dataset */ + if(FAIL == (space_id = H5Dget_space(dset_id))) + goto done; + + /* Get the rank of this dataspace */ + if((sndims = H5Sget_simple_extent_ndims(space_id)) < 0) + goto done; + ndims = (unsigned)sndims; + + /* Verify correct axis */ + if(axis >= ndims) + goto done; + + /* Get the dimensions sizes of the dataspace */ + if(H5Sget_simple_extent_dims(space_id, size, NULL) < 0) + goto done; + + /* Adjust the dimension size of the requested dimension, + but first record the old dimension size */ + old_size = size[axis]; + size[axis] += extension; + if(size[axis] < old_size) + goto done; + + /* Set the extent of the dataset to the new dimension */ + if(H5Dset_extent(dset_id, size) < 0) + goto done; + + /* Get the new dataspace of the dataset */ + if(FAIL == (new_space_id = H5Dget_space(dset_id))) + goto done; + + /* Select a hyperslab corresponding to the append operation */ + for(u = 0 ; u < ndims ; u++) { + start[u] = 0; + stride[u] = 1; + count[u] = size[u]; + block[u] = 1; + if(u == axis) { + count[u] = extension; + start[u] = old_size; + } /* end if */ + } /* end for */ + if(FAIL == H5Sselect_hyperslab(new_space_id, H5S_SELECT_SET, start, stride, count, block)) + goto done; + + /* The # of elemnts in the new extended dataspace */ + if((snelmts = H5Sget_select_npoints(new_space_id)) < 0) + goto done; + nelmts = (hsize_t)snelmts; + + /* create a memory space */ + mem_space_id = H5Screate_simple(1, &nelmts, NULL); + + /* Write the data */ + if(H5Dwrite(dset_id, memtype, mem_space_id, new_space_id, dxpl_id, buf) < 0) + goto done; + + /* Obtain the dataset's access property list */ + if((dapl = H5Dget_access_plist(dset_id)) < 0) + goto done; + + /* Allocate the boundary array */ + boundary = (hsize_t *)HDmalloc(ndims * sizeof(hsize_t)); + + /* Retrieve the append flush property */ + if(H5Pget_append_flush(dapl, ndims, boundary, &append_cb, &udata) < 0) + goto done; + + /* No boundary for this axis */ + if(boundary[axis] == 0) + goto done; + + /* Determine whether a boundary is hit or not */ + for(k = start[axis]; k < size[axis]; k++) + if(!((k + 1) % boundary[axis])) { + hit = TRUE; + break; + } + + if(hit) { /* Hit the boundary */ + /* Invoke callback if there is one */ + if(append_cb && append_cb(dset_id, size, udata) < 0) + goto done; + + /* Do a dataset flush */ + if(H5Dflush(dset_id) < 0) + goto done; + } /* end if */ + + /* Indicate success */ + ret_value = SUCCEED; + +done: + /* Close dxpl if we created it vs. one was passed in */ + if(created_dxpl) { + if(H5Pclose(dxpl_id) < 0) + ret_value = FAIL; + } /* end if */ + + /* Close old dataspace */ + if(space_id != FAIL && H5Sclose(space_id) < 0) + ret_value = FAIL; + + /* Close new dataspace */ + if(new_space_id != FAIL && H5Sclose(new_space_id) < 0) + ret_value = FAIL; + + /* Close memory dataspace */ + if(mem_space_id != FAIL && H5Sclose(mem_space_id) < 0) + ret_value = FAIL; + + /* Close the dataset access property list */ + if(dapl != FAIL && H5Pclose(dapl) < 0) + ret_value = FAIL; + + if(boundary) + HDfree(boundary); + + return ret_value; +} /* H5DOappend() */ + diff --git a/hl/src/H5DOpublic.h b/hl/src/H5DOpublic.h index 9cf6921..1e5eb7a 100644 --- a/hl/src/H5DOpublic.h +++ b/hl/src/H5DOpublic.h @@ -30,6 +30,9 @@ extern "C" { H5_HLDLL herr_t H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, size_t data_size, const void *buf); +H5_HLDLL herr_t H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, + size_t extension, hid_t memtype, const void *buf); + #ifdef __cplusplus } #endif diff --git a/hl/test/Makefile.am b/hl/test/Makefile.am index bea760e..7bfd6b2 100644 --- a/hl/test/Makefile.am +++ b/hl/test/Makefile.am @@ -28,7 +28,8 @@ LDADD=$(LIBH5_HL) $(LIBH5TEST) $(LIBHDF5) # Test programs. These are our main targets. They should be listed in the # order to be executed, generally most specific tests to least specific tests. -TEST_PROG=test_lite test_image test_file_image test_table test_ds test_packet test_dset_opt +TEST_PROG=test_lite test_image test_file_image test_table test_ds test_packet test_dset_opt \ + test_dset_append check_PROGRAMS=$(TEST_PROG) # These programs generate test files for the tests. They don't need to be @@ -46,7 +47,7 @@ endif CHECK_CLEANFILES+=combine_tables[1-2].h5 test_ds[1-9].h5 test_ds10.h5 \ test_image[1-3].h5 file_img[1-2].h5 test_lite[1-4].h5 test_table.h5 \ test_packet_table.h5 test_packet_compress.h5 test_detach.h5 \ - test_dectris.h5 + test_dectris.h5 test_append.h5 # Sources for test_packet executable test_packet_SOURCES=test_packet.c test_packet_vlen.c diff --git a/hl/test/test_dset_append.c b/hl/test/test_dset_append.c new file mode 100644 index 0000000..0f193d9 --- /dev/null +++ b/hl/test/test_dset_append.c @@ -0,0 +1,1196 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Copyright by The HDF Group. * +* Copyright by the Board of Trustees of the University of Illinois. * +* All rights reserved. * +* * +* This file is part of HDF5. The full HDF5 copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the files COPYING and Copyright.html. COPYING can be found at the root * +* of the source code distribution tree; Copyright.html can be found at the * +* root level of an installed copy of the electronic HDF5 document set and * +* is linked from the top-level documents page. It can also be found at * +* http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * +* access to either file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <stdlib.h> +#include <string.h> +#include "h5hltest.h" +#include "H5srcdir.h" +#include "H5DOpublic.h" +#include <math.h> + +#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) +# define H5_ZLIB_HEADER "zlib.h" +#endif +#if defined(H5_ZLIB_HEADER) +# include H5_ZLIB_HEADER /* "zlib.h" */ +#endif + +#define FILE "test_append.h5" +#define DNAME_UNLIM "dataset_unlim" +#define DNAME_LESS "dataset_less" +#define DNAME_VARY "dataset_vary" +#define DNAME_ROW "dataset_row" +#define DNAME_COLUMN "dataset_column" +#define DBUGNAME1 "dataset_bug1" +#define DBUGNAME2 "dataset_bug2" + +/* The callback function for the object flush property */ +static herr_t +flush_func(hid_t H5_ATTR_UNUSED obj_id, void *_udata) +{ + unsigned *flush_ct = (unsigned*)_udata; + ++(*flush_ct); + return 0; +} + +/* The callback function for the append flush property */ +static herr_t +append_func(hid_t H5_ATTR_UNUSED dset_id, hsize_t H5_ATTR_UNUSED *cur_dims, void *_udata) +{ + unsigned *append_ct = (unsigned *)_udata; + ++(*append_ct); + return 0; +} + +/*------------------------------------------------------------------------- + * Function: test_dataset_append_rows_columns + * + * Purpose: Verify that the object flush property and the append flush property + * are working properly when appending rows and columns to a dataset + * with 2 extendible dimensions. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +static int +test_dataset_append_rows_columns(hid_t fid) +{ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t dcpl = -1; /* A copy of dataset creation property */ + hid_t dapl = -1; /* A copy of dataset access property */ + hid_t ffapl = -1; /* The file's file access property list */ + + hsize_t dims[2] = {0, 10}; /* Current dimension sizes */ + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* Maximum dimension sizes */ + hsize_t chunk_dims[2] = {2,5}; /* Chunk dimension sizes */ + int lbuf[10], cbuf[6]; /* The data buffers */ + int buf[6][13], rbuf[6][13]; /* The data buffers */ + + hsize_t boundary[2] = {1, 1}; /* Boundary sizes */ + unsigned append_ct = 0; /* The # of appends */ + unsigned *flush_ptr; /* Points to the flush counter */ + + int i, j; /* Local index variables */ + + TESTING("Append flush with H5DOappend()--append rows & columns"); + + /* Get the file's file access property list */ + if((ffapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Set to create a chunked dataset with 2 extendible dimensions */ + if((sid = H5Screate_simple(2, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* Set append flush property */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_append_flush(dapl, 2, boundary, append_func, &append_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the dataset */ + if((did = H5Dcreate2(fid, DNAME_UNLIM, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR; + + /* Append 6 rows to the dataset */ + for(i = 0; i < 6; i++) { + for(j = 0; j < 10; j++) + lbuf[j] = buf[i][j] = (i * 10) + (j + 1); + if(H5DOappend(did, H5P_DEFAULT, 0, (size_t)1, H5T_NATIVE_INT, lbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 6) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 6) + TEST_ERROR; + + /* Append 3 columns to the dataset */ + for(i = 0; i < 3; i++) { + for(j = 0; j < 6; j++) + cbuf[j] = buf[j][i + 10] = ((i * 6) + (j + 1)) * -1; + if(H5DOappend(did, H5P_DEFAULT, 1, (size_t)1, H5T_NATIVE_INT, cbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 9) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 9) + TEST_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Clear the buffer */ + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + /* Open the dataset again */ + if((did = H5Dopen2(fid, DNAME_UNLIM, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Closing */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(ffapl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl); + H5Pclose(dcpl); + H5Pclose(sid); + H5Dclose(did); + H5Pclose(ffapl); + } H5E_END_TRY; + + return 1; +} /* test_dataset_append_rows_columns() */ + +/*------------------------------------------------------------------------- + * Function: test_dataset_append_rows + * + * Purpose: Verify that the object flush property and the append flush property + * are working properly when appending rows to a dataset with + * one extendible dimension (row). + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +static int +test_dataset_append_rows(hid_t fid) +{ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t dcpl = -1; /* A copy of dataset creation property */ + hid_t dapl = -1; /* A copy of dataset access property */ + hid_t ffapl = -1; /* The file's file access property list */ + + hsize_t dims[2] = {0, 10}; /* Current dimension sizes */ + hsize_t maxdims[2] = {H5S_UNLIMITED, 10}; /* Maximum dimension sizes */ + hsize_t chunk_dims[2] = {2,5}; /* Chunk dimension sizes */ + int lbuf[10]; /* The data buffer */ + int buf[6][10], rbuf[6][10]; /* The data buffers */ + int i, j; /* Local index variables */ + + hsize_t boundary[2] = {1, 0}; /* Boundary sizes */ + unsigned append_ct = 0; /* The # of appends */ + unsigned *flush_ptr; /* Points to the flush counter */ + + TESTING("Append flush with H5DOappend()--append rows"); + + /* Get the file's file access property list */ + if((ffapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Set to create a chunked dataset with 1 extendible dimension */ + if((sid = H5Screate_simple(2, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* Set append flush property */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_append_flush(dapl, 2, boundary, append_func, &append_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the dataset */ + if((did = H5Dcreate2(fid, DNAME_ROW, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR; + + /* Append 6 rows to the dataset */ + for(i = 0; i < 6; i++) { + for(j = 0; j < 10; j++) + lbuf[j] = buf[i][j] = (i * 10) + (j + 1); + if(H5DOappend(did, H5P_DEFAULT, 0, (size_t)1, H5T_NATIVE_INT, lbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 6) + TEST_ERROR; + + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 6) + TEST_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 10; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Clear the buffer */ + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + /* Open the dataset again */ + if((did = H5Dopen2(fid, DNAME_ROW, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 10; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Closing */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(ffapl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl); + H5Pclose(dcpl); + H5Pclose(sid); + H5Dclose(did); + H5Pclose(ffapl); + } H5E_END_TRY; + + return 1; +} /* test_dataset_append_rows() */ + +/*------------------------------------------------------------------------- + * Function: test_dataset_append_columns + * + * Purpose: Verify that the object flush property and the append flush property + * are working properly when appending columns to a dataset + * with one extendible dimension (column). + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +static int +test_dataset_append_columns(hid_t fid) +{ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t dcpl = -1; /* A copy of dataset creation property */ + hid_t dapl = -1; /* A copy of dataset access property */ + hid_t ffapl = -1; /* The file's file access property list */ + + hsize_t dims[2] = {6, 0}; /* Current dimension sizes */ + hsize_t maxdims[2] = {6, H5S_UNLIMITED}; /* Maximum dimension sizes */ + hsize_t chunk_dims[2] = {2,5}; /* Chunk dimension sizes */ + int cbuf[6]; /* The data buffer */ + int buf[6][3], rbuf[6][3]; /* The data buffers */ + int i, j; /* Local index variable */ + + hsize_t boundary[2] = {0, 1}; /* Boundary sizes */ + unsigned append_ct = 0; /* The # of appends */ + unsigned *flush_ptr; /* Points to the flush counter */ + + TESTING("Append flush with H5DOappend()--append columns"); + + /* Get the file's file access property list */ + if((ffapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Set to create a chunked dataset with 1 extendible dimension */ + if((sid = H5Screate_simple(2, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* Set append flush property */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_append_flush(dapl, 2, boundary, append_func, &append_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the dataset */ + if((did = H5Dcreate2(fid, DNAME_COLUMN, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR; + + /* Append 3 columns to the dataset */ + for(i = 0; i < 3; i++) { + for(j = 0; j < 6; j++) + cbuf[j] = buf[j][i] = ((i * 6) + (j + 1)) * -1; + if(H5DOappend(did, H5P_DEFAULT, 1, (size_t)1, H5T_NATIVE_INT, cbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 3) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 3) + TEST_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 3; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Clear the buffer */ + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + /* Open the dataset again */ + if((did = H5Dopen2(fid, DNAME_COLUMN, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 3; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Closing */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(ffapl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl); + H5Pclose(dcpl); + H5Pclose(sid); + H5Dclose(did); + H5Pclose(ffapl); + } H5E_END_TRY; + + return 1; +} /* test_dataset_append_columns() */ + +/*------------------------------------------------------------------------- + * Function: test_dataset_append_BUG1 + * + * Purpose: Verify that the object flush property and the append flush property + * are working properly when appending rows and columns to an + * extendible dataset. + * A BUG occurs: + * when the extendible dataset is set up as follows: + * hsize_t dims[2] = {0, 10}; + * hsize_t maxdims[2] = {H5S_UNLIMITED, 50}; + * when append 6 rows and 3 columns to the dataset; + * The data is correct when the dataset is read at this point; + * The data is incorrect when the dataset is closed, opened again, and read at this point; + * NOTE: the problem does not occur when H5Dflush() is not performed for each row/column. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +static int +test_dataset_append_BUG1(hid_t fid) +{ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t dcpl = -1; /* Dataset creation property */ + hid_t dapl = -1; /* Dataset access property */ + hid_t ffapl = -1; /* The file's file access property list */ + + hsize_t dims[2] = {0, 10}; /* Current dimension sizes */ + hsize_t maxdims[2] = {H5S_UNLIMITED, 50}; /* Maximum dimension sizes */ + hsize_t chunk_dims[2] = {2,5}; /* Chunk dimension sizes */ + int lbuf[10], cbuf[6]; /* The data buffers */ + int buf[6][13], rbuf[6][13]; /* The data buffers */ + int i, j; /* Local index variables */ + + hsize_t boundary[2] = {1, 1}; /* Boundary sizes */ + unsigned append_ct = 0; /* The # of appends */ + unsigned *flush_ptr; /* Points to the flush counter */ + + TESTING("Append flush with H5DOappend()--append rows & columns--BUG1"); + + /* Get the file's file access property list */ + if((ffapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Set to create a chunked dataset with 2 extendible dimensions */ + if((sid = H5Screate_simple(2, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* Set append flush property */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_append_flush(dapl, 2, boundary, append_func, &append_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the dataset */ + if((did = H5Dcreate2(fid, DBUGNAME1, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR; + + /* Append 6 rows to the dataset */ + for(i = 0; i < 6; i++) { + for(j = 0; j < 10; j++) + lbuf[j] = buf[i][j] = (i * 10) + (j + 1); + if(H5DOappend(did, H5P_DEFAULT, 0, (size_t)1, H5T_NATIVE_INT, lbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 6) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 6) + TEST_ERROR; + + /* Append 3 columns to the dataset */ + for(i = 0; i < 3; i++) { + for(j = 0; j < 6; j++) + cbuf[j] = buf[j][i+10] = ((i * 6) + (j + 1)) * -1; + if(H5DOappend(did, H5P_DEFAULT, 1, (size_t)1, H5T_NATIVE_INT, cbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 9) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 9) + TEST_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + +#ifdef BUG1 + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + /* Open the dataset again */ + if((did = H5Dopen(fid, DBUGNAME1, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; +#endif + + /* Closing */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(ffapl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Pclose(dapl); + H5Pclose(sid); + H5Dclose(did); + H5Pclose(ffapl); + } H5E_END_TRY; + + return 1; +} /* test_dataset_append_BUG1() */ + +/*------------------------------------------------------------------------- + * Function: test_dataset_append_BUG2 + * + * Purpose: Verify that the object flush property and the append flush property + * are working properly when appending rows and columns to an + * extendible dataset. + * A BUG occurs: + * when the extendible dataset is set up as follows: + * hsize_t dims[2] = {0, 10}; + * hsize_t maxdims[2] = {50, H5S_UNLIMITED}; + * when append 6 rows and 3 columns to the dataset; + * The data is correct when the dataset is read at this point; + * The data is incorrect when the dataset is closed, opened again, and read at this point; + * NOTE: the problem does not occur when H5Dflush() is not performed for each row/column. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +static int +test_dataset_append_BUG2(hid_t fid) +{ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t dcpl = -1; /* Dataset creation property */ + hid_t dapl = -1; /* Dataset access property */ + hid_t ffapl = -1; /* The file's file access property list */ + + hsize_t dims[2] = {0, 10}; /* Current dimension sizes */ + hsize_t maxdims[2] = {50, H5S_UNLIMITED}; /* Maximum dimension sizes */ + hsize_t chunk_dims[2] = {2,5}; /* Chunk dimension sizes */ + int lbuf[10], cbuf[6]; /* Data buffers */ + int buf[6][13], rbuf[6][13]; /* Data buffers */ + int i, j; /* Local index variables */ + + hsize_t boundary[2] = {1, 1}; /* Boundary sizes */ + unsigned append_ct = 0; /* The # of appends */ + unsigned *flush_ptr; /* Points to the flush counter */ + + TESTING("Append flush with H5DOappend()--append rows & columns--BUG2"); + + /* Get the file's file access property list */ + if((ffapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Set to create a chunked dataset with 2 extendible dimensions */ + if((sid = H5Screate_simple(2, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* Set append flush property */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_append_flush(dapl, 2, boundary, append_func, &append_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the dataset */ + if((did = H5Dcreate2(fid, DBUGNAME2, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR; + + /* Append 6 rows to the dataset */ + for(i = 0; i < 6; i++) { + for(j = 0; j < 10; j++) + lbuf[j] = buf[i][j] = (i * 10) + (j + 1); + if(H5DOappend(did, H5P_DEFAULT, 0, (size_t)1, H5T_NATIVE_INT, lbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 6) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 6) + TEST_ERROR; + + + /* Append 3 columns to the dataset */ + for(i = 0; i < 3; i++) { + for(j = 0; j < 6; j++) + cbuf[j] = buf[j][i+10] = ((i * 6) + (j + 1)) * -1; + if(H5DOappend(did, H5P_DEFAULT, 1, (size_t)1, H5T_NATIVE_INT, cbuf) < 0) + TEST_ERROR; + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 9) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 9) + TEST_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + +#ifdef BUG2 + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + /* Open the dataset again */ + if((did = H5Dopen(fid, DBUGNAME2, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; +#endif + + /* Closing */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(ffapl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Pclose(dapl); + H5Pclose(sid); + H5Dclose(did); + H5Pclose(ffapl); + } H5E_END_TRY; + + return 1; +} /* test_dataset_append_BUG2() */ + + +/*------------------------------------------------------------------------- + * Function: test_dataset_append_less + * + * Purpose: Verify that the object flush property and the append flush property + * are working properly when appending rows and columns to an + * extendible dataset where the append size is less than the boundary + * size. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +static int +test_dataset_append_less(hid_t fid) +{ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t dcpl = -1; /* A copy of dataset creation property */ + hid_t dapl = -1; /* A copy of dataset access property */ + hid_t ffapl = -1; /* The file's file access property list */ + + hsize_t dims[2] = {0, 10}; /* Current dimension sizes */ + hsize_t maxdims[2] = {100, 100}; /* Maximum dimension sizes */ + hsize_t chunk_dims[2] = {2,5}; /* Chunk dimension sizes */ + int lbuf[20], cbuf[6][3]; /* Data buffers */ + int buf[6][13], rbuf[6][13]; /* Data buffers */ + int i, j, k; /* Local index variables */ + + hsize_t boundary[2] = {3, 3}; /* Boundary sizes */ + unsigned append_ct = 0; /* The # of appends */ + unsigned *flush_ptr; /* Points to the flush counter */ + + TESTING("Append flush with H5DOappend()--append size < boundary size"); + + /* Get the file's file access property list */ + if((ffapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Set to create a chunked dataset with 2 extendible dimensions */ + if((sid = H5Screate_simple(2, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* Set append flush property */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_append_flush(dapl, 2, boundary, append_func, &append_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the dataset */ + if((did = H5Dcreate2(fid, DNAME_LESS, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR; + + /* Append to the dataset 2 rows at a time for 3 times */ + for(i = 0, k = 0; i < 6; i++) { + for(j = 0; j < 10; j++, k++) + buf[i][j] = lbuf[k] = (i * 10) + (j + 1); + + if((i + 1) % 2 == 0) { + if(H5DOappend(did, H5P_DEFAULT, 0, (size_t)2, H5T_NATIVE_INT, lbuf) < 0) + TEST_ERROR; + k = 0; + } /* end if */ + } /* end for */ + + /* Verify the # of appends */ + if(append_ct != 2) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 2) + TEST_ERROR; + + /* Append 3 columns to the dataset, once */ + for(i = 0; i < 3; i++) + for(j = 0; j < 6; j++, k++) + cbuf[j][i] = buf[j][i + 10] = ((i * 6) + (j + 1)) * -1; + if(H5DOappend(did, H5P_DEFAULT, 1, (size_t)3, H5T_NATIVE_INT, cbuf) < 0) + TEST_ERROR; + + /* Verify the # of appends */ + if(append_ct != 3) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 3) + TEST_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Clear the buffer */ + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + /* Open the dataset again */ + if((did = H5Dopen2(fid, DNAME_LESS, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Closing */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(ffapl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl); + H5Pclose(dcpl); + H5Pclose(sid); + H5Dclose(did); + H5Pclose(ffapl); + } H5E_END_TRY; + + return 1; +} /* test_dataset_append_less() */ + +/*------------------------------------------------------------------------- + * Function: test_dataset_append_vary + * + * Purpose: Verify that the object flush property and the append flush property + * are working properly when appending rows and columns to an + * extendible dataset where + * row: the append size is 3 times of the boundary size + * the append callback/flush is performed on the 1st boundary hit + * column: the boundary is greater than the append size + * the boundary is not hit at all + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +static int +test_dataset_append_vary(hid_t fid) +{ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t dcpl = -1; /* A copy of dataset creation property */ + hid_t dapl = -1; /* A copy of dataset access property */ + hid_t ffapl = -1; /* The file's file access property list */ + + hsize_t dims[2] = {0, 10}; /* Current dimension sizes */ + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* Maximum dimension sizes */ + hsize_t chunk_dims[2] = {2,5}; /* Chunk dimension sizes */ + int lbuf[60], cbuf[6][3]; /* Data buffers */ + int buf[6][13], rbuf[6][13]; /* Data buffers */ + int i, j, k; /* Local index variables */ + + hsize_t boundary[2] = {3, 7}; /* Boundary sizes */ + unsigned append_ct = 0; /* The # of appends */ + unsigned *flush_ptr; /* Points to the flush counter */ + + TESTING("Append flush with H5DOappend()--append & boundary size vary"); + + /* Get the file's file access property list */ + if((ffapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Set to create a chunked dataset with 2 extendible dimensions */ + if((sid = H5Screate_simple(2, dims, maxdims)) < 0) + FAIL_STACK_ERROR; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR; + + /* Set append flush property */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_append_flush(dapl, 2, boundary, append_func, &append_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the dataset */ + if((did = H5Dcreate2(fid, DNAME_VARY, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR; + + /* Append 6 rows to the dataset, once */ + for(i = 0, k = 0; i < 6; i++) + for(j = 0; j < 10; j++, k++) + buf[i][j] = lbuf[k] = (i * 10) + (j + 1); + if(H5DOappend(did, H5P_DEFAULT, 0, (size_t)6, H5T_NATIVE_INT, lbuf) < 0) + TEST_ERROR; + + /* Verify the # of appends */ + if(append_ct != 1) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 1) + TEST_ERROR; + + /* Append 3 columns to the dataset, once */ + for(i = 0; i < 3; i++) + for(j = 0; j < 6; j++, k++) + cbuf[j][i] = buf[j][i + 10] = ((i * 6) + (j + 1)) * -1; + if(H5DOappend(did, H5P_DEFAULT, 1, (size_t)3, H5T_NATIVE_INT, cbuf) < 0) + TEST_ERROR; + + /* Verify the # of appends */ + if(append_ct != 1) + TEST_ERROR; + + /* Retrieve and verify object flush counts */ + if(H5Pget_object_flush_cb(ffapl, NULL, (void **)&flush_ptr) < 0) + FAIL_STACK_ERROR; + if(*flush_ptr != 1) + TEST_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Clear the dataset */ + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + + /* Open the dataset again */ + if((did = H5Dopen2(fid, DNAME_VARY, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* Read the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) + FAIL_STACK_ERROR; + + /* Verify the data */ + for(i = 0; i < 6; i++) + for(j = 0; j < 13; j++) + if(buf[i][j] != rbuf[i][j]) + TEST_ERROR; + + /* Closing */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(ffapl) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl); + H5Pclose(dcpl); + H5Pclose(sid); + H5Dclose(did); + H5Pclose(ffapl); + } H5E_END_TRY; + + return 1; +} /* test_dataset_append_vary() */ + +/*------------------------------------------------------------------------- + * Function: Main function + * + * Purpose: Test H5Pset/get_object_flush_cb() and H5Pset/get_append_flush() + * along with H5DOappend(). + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; Jan 2014 + * + *------------------------------------------------------------------------- + */ +int main(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + unsigned flush_ct = 0; /* The # of flushes */ + int nerrors = 0; /* The # of errors encountered */ + + /* Get a copy of file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set to use the latest library format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR; + + /* Set object flush property */ + if(H5Pset_object_flush_cb(fapl, flush_func, &flush_ct) < 0) + FAIL_STACK_ERROR; + + /* Create the test file */ + if((fid = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR; + + nerrors += test_dataset_append_rows(fid); + + flush_ct = 0; /* Reset flush counter */ + nerrors += test_dataset_append_columns(fid); + + flush_ct = 0; /* Reset flush counter */ + nerrors += test_dataset_append_rows_columns(fid); + +#ifdef BUG1_BUG2 +/* + * The following tests illustrate the scenarios when H5DOappend does not work with extensible array indexing: + * - when the the dataset has 1 unlimited dimension and the other dimension is fixed but extendible + * - the dataset expands along 1 dimension and then expands along the other dimension + */ + flush_ct = 0; /* Reset flush counter */ + nerrors += test_dataset_append_BUG1(fid); + + flush_ct = 0; /* Reset flush counter */ + nerrors += test_dataset_append_BUG2(fid); +#endif + + flush_ct = 0; /* Reset flush counter */ + nerrors += test_dataset_append_less(fid); + + flush_ct = 0; /* Reset flush counter */ + nerrors += test_dataset_append_vary(fid); + + /* Closing */ + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Check for errors */ + if(nerrors) + goto error; + + return 0; + +error: + return 1; +} + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e425b1..c9fb8c7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -433,6 +433,7 @@ set (H5O_SRCS ${HDF5_SRC_DIR}/H5Odtype.c ${HDF5_SRC_DIR}/H5Oefl.c ${HDF5_SRC_DIR}/H5Ofill.c + ${HDF5_SRC_DIR}/H5Oflush.c ${HDF5_SRC_DIR}/H5Ofsinfo.c ${HDF5_SRC_DIR}/H5Oginfo.c ${HDF5_SRC_DIR}/H5Olayout.c @@ -67,6 +67,9 @@ static herr_t H5AC__check_if_write_permitted(const H5F_t *f, hbool_t *write_permitted_ptr); static herr_t H5AC__ext_config_2_int_config(H5AC_cache_config_t *ext_conf_ptr, H5C_auto_size_ctl_t *int_conf_ptr); +#if H5AC_DO_TAGGING_SANITY_CHECKS +static herr_t H5AC__verify_tag(hid_t dxpl_id, const H5AC_class_t * type); +#endif /* H5AC_DO_TAGGING_SANITY_CHECKS */ /*********************/ @@ -824,6 +827,11 @@ H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t add flags); #endif /* H5AC__TRACE_FILE_ENABLED */ +#if H5AC_DO_TAGGING_SANITY_CHECKS + if(!H5C_get_ignore_tags(f->shared->cache) && (H5AC__verify_tag(dxpl_id, type) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Bad tag value") +#endif /* H5AC_DO_TAGGING_SANITY_CHECKS */ + /* Insert entry into metadata cache */ if(H5C_insert_entry(f, dxpl_id, type, addr, thing, flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, "H5C_insert_entry() failed") @@ -1135,8 +1143,8 @@ H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, size_t trace_entry_size = 0; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ - void * thing = NULL; /* Pointer to native data structure for entry */ - void * ret_value = NULL; /* Return value */ + void * thing = NULL; /* Pointer to native data structure for entry */ + void * ret_value; /* Return value */ FUNC_ENTER_NOAPI(NULL) @@ -1174,6 +1182,11 @@ H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, (int)(type->id), flags); #endif /* H5AC__TRACE_FILE_ENABLED */ +#if H5AC_DO_TAGGING_SANITY_CHECKS + if(!H5C_get_ignore_tags(f->shared->cache) && (H5AC__verify_tag(dxpl_id, type) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, NULL, "Bad tag value") +#endif /* H5AC_DO_TAGGING_SANITY_CHECKS */ + if(NULL == (thing = H5C_protect(f, dxpl_id, type, addr, udata, flags))) HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, "H5C_protect() failed.") @@ -1776,7 +1789,7 @@ H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr, H5AC_cache_config_t *config /* Validate external configuration */ if(H5AC_validate_config(config_ptr) != SUCCEED) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad cache configuration"); + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad cache configuration") if(config_ptr->open_trace_file) { FILE * file_ptr; @@ -2342,6 +2355,7 @@ done: herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag) { + H5C_tag_t tag; /* Tag structure */ H5P_genplist_t *dxpl; /* Dataset transfer property list */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2352,12 +2366,32 @@ H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag) HGOTO_ERROR(H5E_CACHE, H5E_BADTYPE, FAIL, "not a property list") /* Get the current tag value and return that (if prev_tag is NOT null) */ - if(prev_tag) - if((H5P_get(dxpl, "H5AC_metadata_tag", prev_tag)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "unable to query dxpl") + if(prev_tag) { + if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to query dxpl") + *prev_tag = tag.value; + } /* end if */ - /* Set the provided tag value in the dxpl_id. */ - if(H5P_set(dxpl, "H5AC_metadata_tag", &metadata_tag) < 0) + /* Add metadata_tag to tag structure */ + tag.value = metadata_tag; + + /* Determine globality of tag */ + switch(metadata_tag) { + case H5AC__SUPERBLOCK_TAG: + case H5AC__SOHM_TAG: + case H5AC__GLOBALHEAP_TAG: + tag.globality = H5C_GLOBALITY_MAJOR; + break; + case H5AC__FREESPACE_TAG: + tag.globality = H5C_GLOBALITY_MINOR; + break; + default: + tag.globality = H5C_GLOBALITY_NONE; + break; + } /* end switch */ + + /* Set the provided tag in the dxpl_id. */ + if(H5P_set(dxpl, "H5C_tag", &tag) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set property in dxpl") done: @@ -2388,13 +2422,91 @@ H5AC_retag_copied_metadata(const H5F_t *f, haddr_t metadata_tag) HDassert(f); HDassert(f->shared); - /* Call cache-level function to retag entries */ - H5C_retag_copied_metadata(f->shared->cache, metadata_tag); + /* Call cache-level function to re-tag entries with the COPIED tag */ + H5C_retag_entries(f->shared->cache, H5AC__COPIED_TAG, metadata_tag); FUNC_LEAVE_NOAPI(SUCCEED) } /* H5AC_retag_copied_metadata */ +/*------------------------------------------------------------------------------ + * Function: H5AC_flush_tagged_metadata() + * + * Purpose: Wrapper for cache level function which flushes all metadata + * that contains the specific tag. + * + * Return: SUCCEED on success, FAIL otherwise. + * + * Programmer: Mike McGreevy + * May 19, 2010 + * + *------------------------------------------------------------------------------ + */ +herr_t +H5AC_flush_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id) +{ + /* Variable Declarations */ + herr_t ret_value = SUCCEED; + + /* Function Enter Macro */ + FUNC_ENTER_NOAPI(FAIL) + + /* Assertions */ + HDassert(f); + HDassert(f->shared); + + /* Call cache level function to flush metadata entries with specified tag */ + if(H5C_flush_tagged_entries(f, dxpl_id, metadata_tag) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Cannot flush metadata") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_flush_tagged_metadata */ + + +#if H5AC_DO_TAGGING_SANITY_CHECKS + +/*------------------------------------------------------------------------- + * + * Function: H5AC__verify_tag + * + * Purpose: Performs sanity checking on an entry type and tag value + * stored in a supplied dxpl_id. + * + * Return: SUCCEED or FAIL. + * + * Programmer: Mike McGreevy + * October 20, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__verify_tag(hid_t dxpl_id, const H5AC_class_t *type) +{ + H5P_genplist_t *dxpl; /* DXPL for operation */ + H5C_tag_t tag; /* Entry tag to validate */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Get the dataset transfer property list */ + if(NULL == (dxpl = (H5P_genplist_t *)H5I_object_verify(dxpl_id, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + + /* Get the tag from the DXPL */ + if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to query property value") + + /* Verify legal tag value */ + if(H5C_verify_tag(type->id, tag.value, tag.globality) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "tag verification failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__verify_tag */ +#endif /* H5AC_DO_TAGGING_SANITY_CHECKS */ + + /*------------------------------------------------------------------------- * Function: H5AC_get_entry_ring * diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 711d441..5b871ff 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -121,6 +121,12 @@ typedef enum { #define H5AC__DEFAULT_MAX_CACHE_SIZE H5C__DEFAULT_MAX_CACHE_SIZE #define H5AC__DEFAULT_MIN_CLEAN_SIZE H5C__DEFAULT_MIN_CLEAN_SIZE +/* Check if we are sanity checking tagging */ +#if H5C_DO_TAGGING_SANITY_CHECKS +#define H5AC_DO_TAGGING_SANITY_CHECKS 1 +#else +#define H5AC_DO_TAGGING_SANITY_CHECKS 0 +#endif /* * Class methods pertaining to caching. Each type of cached object will @@ -181,10 +187,6 @@ typedef H5C_cache_entry_t H5AC_info_t; /* Typedef for metadata cache (defined in H5Cpkg.h) */ typedef H5C_t H5AC_t; -#define H5AC_METADATA_TAG_NAME "H5AC_metadata_tag" -#define H5AC_METADATA_TAG_SIZE sizeof(haddr_t) -#define H5AC_METADATA_TAG_DEF H5AC__INVALID_TAG - #define H5AC_RING_NAME "H5AC_ring_type" /* Dataset transfer property lists for metadata calls */ @@ -356,6 +358,7 @@ H5_DLL herr_t H5AC_open_trace_file(H5AC_t *cache_ptr, const char *trace_file_nam /* Tag & Ring routines */ H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag); +H5_DLL herr_t H5AC_flush_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id); H5_DLL herr_t H5AC_retag_copied_metadata(const H5F_t *f, haddr_t metadata_tag); H5_DLL herr_t H5AC_ignore_tags(const H5F_t *f); H5_DLL herr_t H5AC_get_entry_ring(const H5F_t *f, haddr_t addr, H5AC_ring_t *ring); @@ -166,28 +166,18 @@ static herr_t H5C_make_space_in_cache(H5F_t * f, size_t space_needed, hbool_t write_permitted); -static herr_t H5C_tag_entry(H5C_t * cache_ptr, +static herr_t H5C__tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id); -static herr_t H5C_flush_tagged_entries(H5F_t * f, - hid_t dxpl_id, - H5C_t * cache_ptr, - haddr_t tag); +static herr_t H5C__mark_tagged_entries(H5C_t * cache_ptr, haddr_t tag); -static herr_t H5C_mark_tagged_entries(H5C_t * cache_ptr, - haddr_t tag); - -static herr_t H5C_flush_marked_entries(H5F_t * f, +static herr_t H5C__flush_marked_entries(H5F_t * f, hid_t dxpl_id); static herr_t H5C__generate_image(const H5F_t *f, H5C_t * cache_ptr, H5C_cache_entry_t *entry_ptr, hid_t dxpl_id, int64_t *entry_size_change_ptr); -#if H5C_DO_TAGGING_SANITY_CHECKS -static herr_t H5C_verify_tag(int id, haddr_t tag); -#endif - #if H5C_DO_SLIST_SANITY_CHECKS static hbool_t H5C_entry_in_skip_list(H5C_t * cache_ptr, H5C_cache_entry_t *target_ptr); @@ -1848,7 +1838,7 @@ H5C_insert_entry(H5F_t * f, entry_ptr->image_up_to_date = FALSE; /* Apply tag to newly inserted entry */ - if(H5C_tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) + if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Cannot tag metadata entry") entry_ptr->is_protected = FALSE; @@ -2777,7 +2767,7 @@ H5C_protect(H5F_t * f, #if H5C_DO_TAGGING_SANITY_CHECKS { - haddr_t tag = HADDR_UNDEF; + H5C_tag_t tag; /* Tag structure */ /* The entry is already in the cache, but make sure that the tag value being passed in via dxpl is still legal. This will ensure that had @@ -2786,14 +2776,14 @@ H5C_protect(H5F_t * f, from disk. */ /* Get the tag from the DXPL */ - if((H5P_get(dxpl, "H5AC_metadata_tag", &tag)) < 0) + if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to query property value"); /* Verify tag value */ if(cache_ptr->ignore_tags != TRUE) { /* Verify legal tag value */ - if((H5C_verify_tag(entry_ptr->type->id, tag)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "tag verification failed"); + if(H5C_verify_tag(entry_ptr->type->id, tag.value, tag.globality) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "tag verification failed") } /* end if */ } #endif @@ -2822,7 +2812,7 @@ H5C_protect(H5F_t * f, #endif /* H5_HAVE_PARALLEL */ /* Apply tag to newly protected entry */ - if(H5C_tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) + if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, NULL, "Cannot tag metadata entry") /* If the entry is very large, and we are configured to allow it, @@ -7462,7 +7452,8 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->in_slist); HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->ring >= ring); + if(!flush_marked_entries || entry_ptr->flush_marker) + HDassert(entry_ptr->ring >= ring); /* increment node pointer now, before we delete its target * from the slist. @@ -7476,7 +7467,8 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); - HDassert(next_entry_ptr->ring >= ring); + if(!flush_marked_entries || next_entry_ptr->flush_marker) + HDassert(next_entry_ptr->ring >= ring); HDassert(entry_ptr != next_entry_ptr); } /* end if */ else @@ -8481,8 +8473,8 @@ H5C_load_entry(H5F_t * f, entry->addr = addr; entry->size = len; HDassert(entry->size < H5C_MAX_ENTRY_SIZE); - entry->compressed = compressed; - entry->compressed_size = compressed_size; + entry->compressed = compressed; + entry->compressed_size = compressed_size; entry->image_ptr = image; entry->image_up_to_date = TRUE; entry->type = type; @@ -9565,7 +9557,34 @@ H5C_ignore_tags(H5C_t * cache_ptr) /*------------------------------------------------------------------------- * - * Function: H5C_tag_entry + * Function: H5C_get_ignore_tags + * + * Purpose: Retrieve the 'ignore_tags' field for the cache + * + * Return: 'ignore_tags' value (can't fail) + * + * Programmer: Quincey Koziol + * April 30, 2016 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5C_get_ignore_tags(const H5C_t *cache_ptr) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + /* Return ignore tag value */ + FUNC_LEAVE_NOAPI(cache_ptr->ignore_tags) +} /* H5C_get_ignore_tags */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C__tag_entry * * Purpose: Tags an entry with the provided tag (contained in the dxpl_id). * If sanity checking is enabled, this function will perform @@ -9582,13 +9601,13 @@ H5C_ignore_tags(H5C_t * cache_ptr) *------------------------------------------------------------------------- */ static herr_t -H5C_tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id) +H5C__tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id) { H5P_genplist_t *dxpl; /* dataset transfer property list */ - haddr_t tag; /* Tag address */ + H5C_tag_t tag; /* Tag structure */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC /* Assertions */ HDassert(cache_ptr != NULL); @@ -9600,13 +9619,13 @@ H5C_tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") /* Get the tag from the DXPL */ - if((H5P_get(dxpl, "H5AC_metadata_tag", &tag)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to query property value") + if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to query property value") if(cache_ptr->ignore_tags != TRUE) { #if H5C_DO_TAGGING_SANITY_CHECKS /* Perform some sanity checks to ensure that a correct tag is being applied */ - if(H5C_verify_tag(entry_ptr->type->id, tag) < 0) + if(H5C_verify_tag(entry_ptr->type->id, tag.value, tag.globality) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "tag verification failed") #endif } else { @@ -9617,70 +9636,30 @@ H5C_tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id) arbitrarily set it to something for the sake of passing the tests. If the tag value is set, then we'll just let it get assigned without additional checking for correctness. */ - if(!tag) - tag = H5AC__IGNORE_TAG; + if(!tag.value) { + tag.value = H5AC__IGNORE_TAG; + tag.globality = H5C_GLOBALITY_NONE; + } /* end if */ } /* end if */ /* Apply the tag to the entry */ - entry_ptr->tag = tag; + entry_ptr->tag = tag.value; -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_tag_entry */ - - -/*------------------------------------------------------------------------- - * - * Function: H5C_flush_tagged_entries - * - * WARNING: Not yet tested or used anywhere. (written awhile ago, - * will keep it around in anticipation of being used in - * subsequent changes to support flushing individual objects). - * - * Purpose: Flushes all entries with the specified tag to disk. - * - * Return: FAIL if error is detected, SUCCEED otherwise. - * - * Programmer: Mike McGreevy - * August 19, 2010 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, H5C_t * cache_ptr, haddr_t tag) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI(FAIL) - - /* Assertions */ - HDassert(0); /* This function is not yet used. We shouldn't be in here yet. */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - - /* Mark all entries with specified tag */ - if(H5C_mark_tagged_entries(cache_ptr, tag) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't mark tagged entries") - - /* Flush all marked entries */ - if(H5C_flush_marked_entries(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush marked entries") + /* Apply the tag globality to the entry */ + entry_ptr->globality = tag.globality; done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_flush_tagged_entries */ +} /* H5C__tag_entry */ + /*------------------------------------------------------------------------- * - * Function: H5C_mark_tagged_entries + * Function: H5C__mark_tagged_entries * - * WARNING: Not yet tested or used anywhere. (written awhile ago, - * will keep it around in anticipation of being used in - * subsequent changes to support flushing individual objects). - * - * Purpose: Set the flush marker on entries in the cache that have - * the specified tag. + * Purpose: Set the flush marker on dirty entries in the cache that have + * the specified tag, as well as all globally tagged entries. * * Return: FAIL if error is detected, SUCCEED otherwise. * @@ -9690,41 +9669,42 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5C_mark_tagged_entries(H5C_t * cache_ptr, haddr_t tag) +H5C__mark_tagged_entries(H5C_t * cache_ptr, haddr_t tag) { - H5C_cache_entry_t *next_entry_ptr; /* entry pointer */ - unsigned u; /* Local index variable */ + unsigned u; /* Local index variable */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC - /* Assertions */ - HDassert(0); /* This function is not yet used. We shouldn't be in here yet. */ - HDassert(cache_ptr != NULL); + /* Sanity check */ + HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - /* Iterate through entries, marking those with specified tag. */ + /* Iterate through hash table entries, marking those with specified tag, as + * well as any major global entries which should always be flushed + * when flushing based on tag value */ for(u = 0; u < H5C__HASH_TABLE_LEN; u++) { - - next_entry_ptr = cache_ptr->index[u]; - while(next_entry_ptr != NULL) { - if(next_entry_ptr->tag == tag) - next_entry_ptr->flush_marker = TRUE; - - next_entry_ptr = next_entry_ptr->ht_next; - } /* end while */ - } /* for */ + H5C_cache_entry_t *entry_ptr; /* Entry pointer */ + + entry_ptr = cache_ptr->index[u]; + while(entry_ptr != NULL) { + if((entry_ptr->tag == tag) || (entry_ptr->globality == H5C_GLOBALITY_MAJOR)) { + /* We only want to set the flush marker on entries that + * actually need flushed (i.e., dirty ones) */ + if(entry_ptr->is_dirty) + entry_ptr->flush_marker = TRUE; + } /* end if */ + + entry_ptr = entry_ptr->ht_next; + } /* end while */ + } /* end for */ FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5C_mark_tagged_entries */ +} /* H5C__mark_tagged_entries */ /*------------------------------------------------------------------------- * - * Function: H5C_flush_marked_entries - * - * WARNING: Not yet tested or used anywhere. (written awhile ago, - * will keep it around in anticipation of being used in - * subsequent changes to support flushing individual objects). + * Function: H5C__flush_marked_entries * * Purpose: Flushes all marked entries in the cache. * @@ -9736,14 +9716,13 @@ H5C_mark_tagged_entries(H5C_t * cache_ptr, haddr_t tag) *------------------------------------------------------------------------- */ static herr_t -H5C_flush_marked_entries(H5F_t * f, hid_t dxpl_id) +H5C__flush_marked_entries(H5F_t * f, hid_t dxpl_id) { herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Assertions */ - HDassert(0); /* This function is not yet used. We shouldn't be in here yet. */ HDassert(f != NULL); /* Flush all marked entries */ @@ -9752,7 +9731,7 @@ H5C_flush_marked_entries(H5F_t * f, hid_t dxpl_id) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_flush_marked_entries */ +} /* H5C__flush_marked_entries */ #if H5C_DO_TAGGING_SANITY_CHECKS @@ -9769,8 +9748,8 @@ done: * *------------------------------------------------------------------------- */ -static herr_t -H5C_verify_tag(int id, haddr_t tag) +herr_t +H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality) { herr_t ret_value = SUCCEED; @@ -9784,7 +9763,6 @@ H5C_verify_tag(int id, haddr_t tag) else if(tag == H5AC__INVALID_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "no metadata tag provided") else { - /* Perform some sanity checks on tag value. Certain entry * types require certain tag values, so check that these * constraints are met. */ @@ -9793,37 +9771,45 @@ H5C_verify_tag(int id, haddr_t tag) if((id == H5AC_SUPERBLOCK_ID) || (id == H5AC_DRVRINFO_ID)) { if(tag != H5AC__SUPERBLOCK_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "superblock not tagged with H5AC__SUPERBLOCK_TAG") - } + if(globality != H5C_GLOBALITY_MAJOR) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "superblock/driver-info globality not marked with H5C_GLOBALITY_MAJOR") + } /* end if */ else { if(tag == H5AC__SUPERBLOCK_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "H5AC__SUPERBLOCK_TAG applied to non-superblock entry") - } + } /* end else */ /* Free Space Manager */ if((id == H5AC_FSPACE_HDR_ID) || (id == H5AC_FSPACE_SINFO_ID)) { if(tag != H5AC__FREESPACE_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "freespace entry not tagged with H5AC__FREESPACE_TAG") - } + if(globality != H5C_GLOBALITY_MINOR) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "freespace entry globality not marked with H5C_GLOBALITY_MINOR") + } /* end if */ else { if(tag == H5AC__FREESPACE_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "H5AC__FREESPACE_TAG applied to non-freespace entry") - } + } /* end else */ /* SOHM */ if((id == H5AC_SOHM_TABLE_ID) || (id == H5AC_SOHM_LIST_ID)) { if(tag != H5AC__SOHM_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "sohm entry not tagged with H5AC__SOHM_TAG") - } + if(globality != H5C_GLOBALITY_MAJOR) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "sohm entry globality not marked with H5C_GLOBALITY_MAJOR") + } /* end if */ /* Global Heap */ if(id == H5AC_GHEAP_ID) { if(tag != H5AC__GLOBALHEAP_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "global heap not tagged with H5AC__GLOBALHEAP_TAG") - } + if(globality != H5C_GLOBALITY_MAJOR) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "global heap entry globality not marked with H5C_GLOBALITY_MAJOR") + } /* end if */ else { if(tag == H5AC__GLOBALHEAP_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "H5AC__GLOBALHEAP_TAG applied to non-globalheap entry") - } + } /* end else */ } /* end else */ done: @@ -9834,11 +9820,53 @@ done: /*------------------------------------------------------------------------- * - * Function: H5C_retag_copied_metadata + * Function: H5C_flush_tagged_entries + * + * Purpose: Flushes all entries with the specified tag to disk. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: Mike McGreevy + * August 19, 2010 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag) +{ + /* Variable Declarations */ + H5C_t *cache_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Assertions */ + HDassert(f); + HDassert(f->shared); + + /* Get cache pointer */ + cache_ptr = f->shared->cache; + + /* Mark all entries with specified tag */ + if(H5C__mark_tagged_entries(cache_ptr, tag) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't mark tagged entries") + + /* Flush all marked entries */ + if(H5C__flush_marked_entries(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush marked entries") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_flush_tagged_entries */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C_retag_entries * * Purpose: Searches through cache index for all entries with the - * H5AC__COPIED_TAG, indicating that it was created as a - * result of an object copy, and applies the provided tag. + * value specified by src_tag and changes it to the value + * specified by dest_tag. * * Return: SUCCEED or FAIL. * @@ -9848,30 +9876,29 @@ done: *------------------------------------------------------------------------- */ void -H5C_retag_copied_metadata(H5C_t * cache_ptr, haddr_t metadata_tag) +H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest_tag) { unsigned u; /* Local index variable */ FUNC_ENTER_NOAPI_NOINIT_NOERR + /* Sanity check */ HDassert(cache_ptr); - /* Iterate through entries, retagging those with the H5AC__COPIED_TAG tag */ + /* Iterate through entries, retagging those with the src_tag tag */ for(u = 0; u < H5C__HASH_TABLE_LEN; u++) { - H5C_cache_entry_t *next_entry_ptr; /* entry pointer */ - - next_entry_ptr = cache_ptr->index[u]; - while(next_entry_ptr != NULL) { - if(cache_ptr->index[u] != NULL) - if((cache_ptr->index[u])->tag == H5AC__COPIED_TAG) - (cache_ptr->index[u])->tag = metadata_tag; - - next_entry_ptr = next_entry_ptr->ht_next; - } /* end while */ + H5C_cache_entry_t *entry_ptr; /* entry pointer */ + + entry_ptr = cache_ptr->index[u]; + while(entry_ptr) { + if(entry_ptr->tag == src_tag) + entry_ptr->tag = dest_tag; + entry_ptr = entry_ptr->ht_next; + } /* end while */ } /* end for */ FUNC_LEAVE_NOAPI_VOID -} /* H5C_retag_copied_metadata */ +} /* H5C_retag_entries */ /*------------------------------------------------------------------------- diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index a062214..d43c845 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -205,12 +205,17 @@ #define H5C__FLUSH_MARKED_ENTRIES_FLAG 0x0080 #define H5C__FLUSH_IGNORE_PROTECTED_FLAG 0x0100 #define H5C__READ_ONLY_FLAG 0x0200 -#define H5C__FREE_FILE_SPACE_FLAG 0x0800 -#define H5C__TAKE_OWNERSHIP_FLAG 0x1000 -#define H5C__FLUSH_LAST_FLAG 0x2000 -#define H5C__FLUSH_COLLECTIVELY_FLAG 0x4000 +#define H5C__FREE_FILE_SPACE_FLAG 0x0400 +#define H5C__TAKE_OWNERSHIP_FLAG 0x0800 +#define H5C__FLUSH_LAST_FLAG 0x1000 +#define H5C__FLUSH_COLLECTIVELY_FLAG 0x2000 #define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x8000 +/* Definitions for cache "tag" property */ +#define H5C_TAG_NAME "H5C_tag" +#define H5C_TAG_SIZE sizeof(H5C_tag_t) +#define H5C_TAG_DEF {(haddr_t)0, H5C_GLOBALITY_NONE} + /* Debugging/sanity checking/statistics settings */ #ifndef NDEBUG #define H5C_DO_SANITY_CHECKS 1 @@ -268,8 +273,20 @@ /* Typedef for the main structure for the cache (defined in H5Cpkg.h) */ typedef struct H5C_t H5C_t; +/* Define enum for cache entry tag 'globality' value */ +typedef enum { + H5C_GLOBALITY_NONE=0, /* Non-global tag */ + H5C_GLOBALITY_MINOR, /* global, not flushed during single object flush */ + H5C_GLOBALITY_MAJOR /* global, needs flushed during single obect flush */ +} H5C_tag_globality_t; + +/* Cache entry tag structure */ +typedef struct H5C_tag_t { + haddr_t value; + H5C_tag_globality_t globality; +} H5C_tag_t; -/*************************************************************************** +/* * * Struct H5C_class_t * @@ -1596,6 +1613,7 @@ typedef struct H5C_cache_entry_t { hbool_t image_up_to_date; const H5C_class_t * type; haddr_t tag; + H5C_tag_globality_t globality; hbool_t is_dirty; hbool_t dirtied; hbool_t is_protected; @@ -1951,6 +1969,10 @@ H5_DLL herr_t H5C_dest(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, unsigned flags); H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags); +H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); +#if H5C_DO_TAGGING_SANITY_CHECKS +herr_t H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality); +#endif H5_DLL herr_t H5C_flush_to_min_clean(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5C_get_cache_auto_resize_config(const H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_ptr); @@ -1992,7 +2014,8 @@ H5_DLL herr_t H5C_unprotect(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *thing, H5_DLL herr_t H5C_validate_resize_config(H5C_auto_size_ctl_t *config_ptr, unsigned int tests); H5_DLL herr_t H5C_ignore_tags(H5C_t *cache_ptr); -H5_DLL void H5C_retag_copied_metadata(H5C_t *cache_ptr, haddr_t metadata_tag); +H5_DLL hbool_t H5C_get_ignore_tags(const H5C_t *cache_ptr); +H5_DLL void H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest_tag); H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring); #ifdef H5_HAVE_PARALLEL @@ -917,3 +917,41 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Dset_extent() */ + +/*------------------------------------------------------------------------- + * Function: H5Dflush + * + * Purpose: Flushes all buffers associated with a dataset. + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Mike McGreevy + * May 19, 2010 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dflush(hid_t dset_id) +{ + H5D_t *dset; /* Dataset for this operation */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", dset_id); + + /* Check args */ + if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + + /* Flush any dataset information still cached in memory */ + if(H5D__flush_real(dset, H5AC_ind_read_dxpl_id) < 0) + HDONE_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to flush cached dataset info") + + /* Flush object's metadata to file */ + if(H5O_flush_common(&dset->oloc, dset_id, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush dataset and object flush callback") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Dflush */ + diff --git a/src/H5Dint.c b/src/H5Dint.c index 0355656..a5d214e 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -68,7 +68,7 @@ static herr_t H5D_build_extfile_prefix(const H5D_t *dset, hid_t dapl_id, static herr_t H5D__open_oid(H5D_t *dataset, hid_t dapl_id, hid_t dxpl_id); static herr_t H5D__init_storage(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_t old_dim[]); - +static herr_t H5D__append_flush_setup(H5D_t *dset, hid_t dapl_id); /*********************/ /* Package Variables */ @@ -1247,6 +1247,10 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, /* Indicate that the layout information was initialized */ layout_init = TRUE; + /* Set up append flush parameters for the dataset */ + if(H5D__append_flush_setup(new_dset, dapl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set up flush append property") + /* Set the external file prefix */ if(H5D_build_extfile_prefix(new_dset, dapl_id, &new_dset->shared->extfile_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize external file prefix") @@ -1491,6 +1495,85 @@ done: } /* end H5D_open() */ +/* + *------------------------------------------------------------------------- + * Function: H5D__flush_append_setup + * + * Purpose: Set the append flush parameters for a dataset + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Wednesday, January 8, 2014 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__append_flush_setup(H5D_t *dset, hid_t dapl_id) +{ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_STATIC + + /* Check args */ + HDassert(dset); + HDassert(dset->shared); + + /* Set default append flush values */ + HDmemset(&dset->shared->append_flush, 0, sizeof(dset->shared->append_flush)); + + /* If the dataset is chunked and there is a non-default DAPL */ + if(dapl_id != H5P_DATASET_ACCESS_DEFAULT && dset->shared->layout.type == H5D_CHUNKED) { + H5P_genplist_t *dapl; /* data access property list object pointer */ + + /* Get dataset access property list */ + if(NULL == (dapl = (H5P_genplist_t *)H5I_object(dapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for dapl ID"); + + /* Check if append flush property exists */ + if(H5P_exist_plist(dapl, H5D_ACS_APPEND_FLUSH_NAME) > 0) { + H5D_append_flush_t info; + + /* Get append flush property */ + if(H5P_get(dapl, H5D_ACS_APPEND_FLUSH_NAME, &info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get append flush info") + if(info.ndims > 0) { + hsize_t curr_dims[H5S_MAX_RANK]; /* current dimension sizes */ + hsize_t max_dims[H5S_MAX_RANK]; /* current dimension sizes */ + int rank; /* dataspace # of dimensions */ + unsigned u; /* local index variable */ + + /* Get dataset rank */ + if((rank = H5S_get_simple_extent_dims(dset->shared->space, curr_dims, max_dims)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions") + if(info.ndims != (unsigned)rank) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "boundary dimension rank does not match dataset rank") + + /* Validate boundary sizes */ + for(u = 0; u < info.ndims; u++) + if(info.boundary[u] != 0) /* when a non-zero boundary is set */ + /* the dimension is extendible? */ + if(max_dims[u] != H5S_UNLIMITED && max_dims[u] == curr_dims[u]) + break; + + /* At least one boundary dimension is not extendible */ + if(u != info.ndims) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "boundary dimension is not valid") + + /* Copy append flush settings */ + dset->shared->append_flush.ndims = info.ndims; + dset->shared->append_flush.func = info.func; + dset->shared->append_flush.udata = info.udata; + HDmemcpy(dset->shared->append_flush.boundary, info.boundary, sizeof(info.boundary)); + } /* end if */ + } /* end if */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__append_flush_setup() */ + + /*------------------------------------------------------------------------- * Function: H5D__open_oid * @@ -1555,6 +1638,10 @@ H5D__open_oid(H5D_t *dataset, hid_t dapl_id, hid_t dxpl_id) /* Indicate that the layout information was initialized */ layout_init = TRUE; + /* Set up flush append property */ + if(H5D__append_flush_setup(dataset, dapl_id)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to set up flush append property") + /* Point at dataset's copy, to cache it for later */ fill_prop = &dataset->shared->dcpl_cache.fill; @@ -3083,28 +3170,28 @@ H5D_get_access_plist(H5D_t *dset) FUNC_ENTER_NOAPI_NOINIT /* Make a copy of the default dataset access property list */ - if (NULL == (old_plist = (H5P_genplist_t *)H5I_object(H5P_LST_DATASET_ACCESS_ID_g))) + if(NULL == (old_plist = (H5P_genplist_t *)H5I_object(H5P_LST_DATASET_ACCESS_ID_g))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") - if ((new_dapl_id = H5P_copy_plist(old_plist, TRUE)) < 0) + if((new_dapl_id = H5P_copy_plist(old_plist, TRUE)) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_CANTINIT, FAIL, "can't copy dataset access property list") - if (NULL == (new_plist = (H5P_genplist_t *)H5I_object(new_dapl_id))) + if(NULL == (new_plist = (H5P_genplist_t *)H5I_object(new_dapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") - /* If the dataset is chunked then copy the rdcc parameters */ - if (dset->shared->layout.type == H5D_CHUNKED) { - if (H5P_set(new_plist, H5D_ACS_DATA_CACHE_NUM_SLOTS_NAME, &(dset->shared->cache.chunk.nslots)) < 0) + /* If the dataset is chunked then copy the rdcc & append flush parameters */ + if(dset->shared->layout.type == H5D_CHUNKED) { + if(H5P_set(new_plist, H5D_ACS_DATA_CACHE_NUM_SLOTS_NAME, &(dset->shared->cache.chunk.nslots)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set data cache number of slots") - if (H5P_set(new_plist, H5D_ACS_DATA_CACHE_BYTE_SIZE_NAME, &(dset->shared->cache.chunk.nbytes_max)) < 0) + if(H5P_set(new_plist, H5D_ACS_DATA_CACHE_BYTE_SIZE_NAME, &(dset->shared->cache.chunk.nbytes_max)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set data cache byte size") - if (H5P_set(new_plist, H5D_ACS_PREEMPT_READ_CHUNKS_NAME, &(dset->shared->cache.chunk.w0)) < 0) + if(H5P_set(new_plist, H5D_ACS_PREEMPT_READ_CHUNKS_NAME, &(dset->shared->cache.chunk.w0)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set preempt read chunks") + if(H5P_set(new_plist, H5D_ACS_APPEND_FLUSH_NAME, &dset->shared->append_flush) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set append flush property") } /* end if */ - /* Set the VDS view option */ + /* Set the VDS view & printf gap options */ if(H5P_set(new_plist, H5D_ACS_VDS_VIEW_NAME, &(dset->shared->layout.storage.u.virt.view)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set VDS view") - - /* Set the VDS printf gap option */ if(H5P_set(new_plist, H5D_ACS_VDS_PRINTF_GAP_NAME, &(dset->shared->layout.storage.u.virt.printf_gap)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set VDS printf gap") diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index f582367..c0b8b28 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -465,6 +465,7 @@ typedef struct H5D_shared_t { H5D_rdcc_t chunk; /* Information about chunked data */ } cache; + H5D_append_flush_t append_flush; /* Append flush property information */ char *extfile_prefix; /* expanded external file prefix */ } H5D_shared_t; diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index ab60a50..d1468dd 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -54,6 +54,7 @@ #define H5D_ACS_PREEMPT_READ_CHUNKS_NAME "rdcc_w0" /* Preemption read chunks first */ #define H5D_ACS_VDS_VIEW_NAME "vds_view" /* VDS view option */ #define H5D_ACS_VDS_PRINTF_GAP_NAME "vds_printf_gap" /* VDS printf gap size */ +#define H5D_ACS_APPEND_FLUSH_NAME "append_flush" /* Append flush actions */ #define H5D_ACS_EFILE_PREFIX_NAME "external file prefix" /* External file prefix */ /* ======== Data transfer properties ======== */ @@ -151,6 +152,14 @@ typedef struct H5D_copy_file_ud_t { H5T_t *src_dtype; /* Copy of datatype for dataset */ } H5D_copy_file_ud_t; +/* Structure for dataset append flush property (H5Pset_append_flush) */ +typedef struct H5D_append_flush_t { + unsigned ndims; /* The # of dimensions for "boundary" */ + hsize_t boundary[H5S_MAX_RANK]; /* The dimension sizes for determining boundary */ + H5D_append_cb_t func; /* The callback function */ + void *udata; /* User data */ +} H5D_append_flush_t; + /*****************************/ /* Library Private Variables */ diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 0b6540c..e1555b9 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -109,6 +109,9 @@ typedef enum H5D_vds_view_t { H5D_VDS_LAST_AVAILABLE = 1 } H5D_vds_view_t; +/* Callback for H5Pset_append_flush() in a dataset access property list */ +typedef herr_t (*H5D_append_cb_t)(hid_t dataset_id, hsize_t *cur_dims, void *op_data); + /********************/ /* Public Variables */ /********************/ @@ -157,6 +160,7 @@ H5_DLL herr_t H5Dvlen_get_buf_size(hid_t dataset_id, hid_t type_id, hid_t space_ H5_DLL herr_t H5Dfill(const void *fill, hid_t fill_type, void *buf, hid_t buf_type, hid_t space); H5_DLL herr_t H5Dset_extent(hid_t dset_id, const hsize_t size[]); +H5_DLL herr_t H5Dflush(hid_t dset_id); H5_DLL herr_t H5Dscatter(H5D_scatter_func_t op, void *op_data, hid_t type_id, hid_t dst_space_id, void *dst_buf); H5_DLL herr_t H5Dgather(hid_t src_space_id, const void *src_buf, hid_t type_id, diff --git a/src/H5Fint.c b/src/H5Fint.c index f4e6550..9ca472d 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -171,6 +171,9 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) latest_format = TRUE; if(H5P_set(new_plist, H5F_ACS_LATEST_FORMAT_NAME, &latest_format) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set 'latest format' flag") + if(H5P_set(new_plist, H5F_ACS_OBJECT_FLUSH_CB_NAME, &(f->shared->object_flush)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set object flush callback") + if(f->shared->efc) efc_size = H5F_efc_max_nfiles(f->shared->efc); if(H5P_set(new_plist, H5F_ACS_EFC_SIZE_NAME, &efc_size) < 0) @@ -679,6 +682,10 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t */ f->shared->use_tmp_space = !H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI); + /* Get object flush callback information */ + if(H5P_get(plist, H5F_ACS_OBJECT_FLUSH_CB_NAME, &(f->shared->object_flush)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get object flush cb info") + /* * Create a metadata cache with the specified number of elements. * The cache might be created with a different number of elements and @@ -2070,6 +2077,39 @@ done: /*------------------------------------------------------------------------- + * Function: H5F_object_flush_cb + * + * Purpose: To invoke the callback function for object flush that is set + * in the file's access property list. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Vailin Choi; October 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_object_flush_cb(H5F_t *f, hid_t obj_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(f); + HDassert(f->shared); + + /* Invoke object flush callback if there is one */ + if(f->shared->object_flush.func && f->shared->object_flush.func(obj_id, f->shared->object_flush.udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "object flush callback returns error") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F_object_flush_cb() */ + + +/*------------------------------------------------------------------------- * Function: H5F__set_base_addr * * Purpose: Quick and dirty routine to set the file's 'base_addr' value diff --git a/src/H5Fio.c b/src/H5Fio.c index d001bc0..1fb3459 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -199,3 +199,48 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_block_write() */ + +/*------------------------------------------------------------------------- + * Function: H5F_flush_tagged_metadata + * + * Purpose: Flushes metadata with specified tag in the metadata cache + * to disk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mike McGreevy + * September 9, 2010 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id) +{ + H5F_io_info_t fio_info; /* I/O info for operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Use tag to search for and flush associated metadata */ + if(H5AC_flush_tagged_metadata(f, tag, dxpl_id)<0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") + + /* Set up I/O info for operation */ + fio_info.f = f; + + if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + + + /* Flush and reset the accumulator */ + if(H5F__accum_reset(&fio_info, TRUE) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") + + /* Flush file buffers to disk. */ + if(H5FD_flush(f->shared->lf, dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "low level flush failed") + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5F_flush_tagged_metadata */ + diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 06f0207..e2874d1 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -293,6 +293,9 @@ struct H5F_file_t { /* Metadata accumulator information */ H5F_meta_accum_t accum; /* Metadata accumulator info */ + + /* Object flush info */ + H5F_object_flush_t object_flush; /* Information for object flush callback */ }; /* diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 05c70d1..93965e0 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -459,6 +459,7 @@ #define H5F_ACS_MULTI_TYPE_NAME "multi_type" /* Data type in multi file driver */ #define H5F_ACS_LATEST_FORMAT_NAME "latest_format" /* 'Use latest format version' flag */ #define H5F_ACS_WANT_POSIX_FD_NAME "want_posix_fd" /* Internal: query the file descriptor from the core VFD, instead of the memory address */ +#define H5F_ACS_OBJECT_FLUSH_CB_NAME "object_flush_cb" /* Object flush callback */ #define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */ #define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */ #define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */ @@ -591,6 +592,12 @@ typedef struct H5F_file_t H5F_file_t; /* Block aggregation structure */ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; +/* Structure for object flush callback property (H5Pset_object_flush_cb)*/ +typedef struct H5F_object_flush_t { + H5F_flush_cb_t func; /* The callback function */ + void *udata; /* User data */ +} H5F_object_flush_t; + /* I/O Info for an operation */ typedef struct H5F_io_info_t { const H5F_t *f; /* File object */ @@ -688,6 +695,12 @@ H5_DLL herr_t H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, H5_DLL herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, hid_t dxpl_id, const void *buf); +/* Functions that flush or evict */ +H5_DLL herr_t H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id); + +/* Routine to invoke callback function upon object flush */ +H5_DLL herr_t H5F_object_flush_cb(H5F_t *f, hid_t obj_id); + /* Address-related functions */ H5_DLL void H5F_addr_encode(const H5F_t *f, uint8_t **pp, haddr_t addr); H5_DLL void H5F_addr_encode_len(size_t addr_len, uint8_t **pp, haddr_t addr); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index eba9b12..6fa18e1 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -175,6 +175,9 @@ typedef enum H5F_file_space_type_t { H5F_FILE_SPACE_NTYPES /* must be last */ } H5F_file_space_type_t; +/* Callback for H5Pset_object_flush_cb() in a file access property list */ +typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); + #ifdef __cplusplus extern "C" { diff --git a/src/H5Oflush.c b/src/H5Oflush.c new file mode 100644 index 0000000..0bec096 --- /dev/null +++ b/src/H5Oflush.c @@ -0,0 +1,132 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Oflush.c + * Aug 19, 2010 + * Mike McGreevy <mamcgree@hdfgroup.org> + * + * Purpose: Object flush/refresh routines. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Omodule.h" /* This source code file is part of the H5O module */ + +/***********/ +/* Headers */ +/***********/ + +#include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Datasets */ +#include "H5Eprivate.h" /* Errors */ +#include "H5Fprivate.h" /* Files */ +#include "H5Gprivate.h" /* Groups */ +#include "H5Iprivate.h" /* IDs */ +#include "H5Opkg.h" /* Objects */ + +/********************/ +/* Local Prototypes */ +/********************/ +static herr_t H5O_oh_tag(const H5O_loc_t *oloc, hid_t dxpl_id, haddr_t *tag); + +/*************/ +/* Functions */ +/*************/ + + + +/*------------------------------------------------------------------------- + * Function: H5O_flush_common + * + * Purpose: Flushes the object's metadata + * Invokes the user-defined callback if there is one. + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Vailin Choi; Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_flush_common(H5O_loc_t *oloc, hid_t obj_id, hid_t dxpl_id) +{ + haddr_t tag = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Retrieve tag for object */ + if(H5O_oh_tag(oloc, dxpl_id, &tag) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFLUSH, FAIL, "unable to flush object metadata") + + /* Flush metadata based on tag value of the object */ + if(H5F_flush_tagged_metadata(oloc->file, tag, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") + + /* Check to invoke callback */ + if(H5F_object_flush_cb(oloc->file, obj_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFLUSH, FAIL, "unable to do object flush callback") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_flush_common() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_oh_tag + * + * Purpose: Get object header's address--tag value for the object + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Mike McGreevy + * May 19, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_oh_tag(const H5O_loc_t *oloc, hid_t dxpl_id, haddr_t *tag) +{ + H5O_t *oh = NULL; /* Object header */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check args */ + HDassert(oloc); + + /* Get object header for object */ + if(NULL == (oh = H5O_protect(oloc, dxpl_id, H5AC__READ_ONLY_FLAG))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object's object header") + + /* Get object header's address (i.e. the tag value for this object) */ + if(HADDR_UNDEF == (*tag = H5O_OH_GET_ADDR(oh))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to get address of object header") + +done: + /* Unprotect object header on failure */ + if(oh && H5O_unprotect(oloc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_oh_tag() */ + diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 17ab665..9ad8662 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -902,6 +902,9 @@ H5_DLL int H5O_msg_get_chunkno(const H5O_loc_t *loc, unsigned type_id, hid_t dxp H5_DLL herr_t H5O_msg_lock(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id); H5_DLL herr_t H5O_msg_unlock(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id); +/* Object metadata flush/refresh routines */ +H5_DLL herr_t H5O_flush_common(H5O_loc_t *oloc, hid_t obj_id, hid_t dxpl_id); + /* Object copying routines */ H5_DLL herr_t H5O_copy_header_map(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, hid_t dxpl_id, H5O_copy_t *cpy_info, hbool_t inc_depth, diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index 6407d4c..aa58dc4 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -73,6 +73,9 @@ #define H5D_ACS_VDS_PRINTF_GAP_DEF (hsize_t)0 #define H5D_ACS_VDS_PRINTF_GAP_ENC H5P__encode_hsize_t #define H5D_ACS_VDS_PRINTF_GAP_DEC H5P__decode_hsize_t +/* Definition for append flush */ +#define H5D_ACS_APPEND_FLUSH_SIZE sizeof(H5D_append_flush_t) +#define H5D_ACS_APPEND_FLUSH_DEF {0,{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0},NULL,NULL} /* Definitions for external file prefix */ #define H5D_ACS_EFILE_PREFIX_SIZE sizeof(char *) #define H5D_ACS_EFILE_PREFIX_DEF NULL /*default is no prefix */ @@ -157,9 +160,9 @@ const H5P_libclass_t H5P_CLS_DACC[1] = {{ /*******************/ /* Property value defaults */ +static const H5D_append_flush_t H5D_def_append_flush_g = H5D_ACS_APPEND_FLUSH_DEF; /* Default setting for append flush */ static const char *H5D_def_efile_prefix_g = H5D_ACS_EFILE_PREFIX_DEF; /* Default external file prefix string */ - /*------------------------------------------------------------------------- * Function: H5P__dacc_reg_prop @@ -212,6 +215,12 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register info for append flush */ + /* (Note: this property should not have an encode/decode callback -QAK) */ + if(H5P_register_real(pclass, H5D_ACS_APPEND_FLUSH_NAME, H5D_ACS_APPEND_FLUSH_SIZE, &H5D_def_append_flush_g, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register property for external file prefix */ if(H5P_register_real(pclass, H5D_ACS_EFILE_PREFIX_NAME, H5D_ACS_EFILE_PREFIX_SIZE, &H5D_def_efile_prefix_g, NULL, H5D_ACS_EFILE_PREFIX_SET, H5D_ACS_EFILE_PREFIX_GET, H5D_ACS_EFILE_PREFIX_ENC, H5D_ACS_EFILE_PREFIX_DEC, @@ -1076,6 +1085,125 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pset_append_flush + * + * Purpose: Sets the boundary, callback function, and user data in the + * property list. + * "ndims": number of array elements for boundary + * "boundary": used to determine whether the current dimension hits + * a boundary; if so, invoke the callback function and + * flush the dataset. + * "func": the callback function to invoke when the boundary is hit + * "udata": the user data to pass as parameter with the callback function + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_append_flush(hid_t plist_id, unsigned ndims, const hsize_t *boundary, H5D_append_cb_t func, void *udata) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5D_append_flush_t info; /* Property for append flush parameters */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "iIu*hx*x", plist_id, ndims, boundary, func, udata); + + /* Check arguments */ + if(0 == ndims) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dimensionality cannot be zero") + if(ndims > H5S_MAX_RANK) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dimensionality is too large") + if(!boundary) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no boundary dimensions specified") + + /* Check if the callback function is NULL and the user data is non-NULL. + * This is almost certainly an error as the user data will not be used. */ + if(!func && udata) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "callback is NULL while user data is not") + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set up values */ + info.ndims = ndims; + info.func = func; + info.udata = udata; + + HDmemset(info.boundary, 0, sizeof(info.boundary)); + /* boundary can be 0 to indicate no boundary is set */ + for(u = 0; u < ndims; u++) { + if(boundary[u] != (boundary[u] & 0xffffffff)) /* negative value (including H5S_UNLIMITED) */ + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all boundary dimensions must be less than 2^32") + info.boundary[u] = boundary[u]; /* Store user's boundary dimensions */ + } /* end for */ + + /* Set values */ + if(H5P_set(plist, H5D_ACS_APPEND_FLUSH_NAME, &info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set append flush") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_append_flush() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_append_flush() + * + * Purpose: Retrieves the boundary, callback function and user data set in + * property list. + * Note that the # of boundary sizes to retrieve will not exceed + * the parameter "ndims" and the ndims set previously via + * H5Pset_append_flush(). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_append_flush(hid_t plist_id, unsigned ndims, hsize_t boundary[], H5D_append_cb_t *func, void **udata) +{ + H5P_genplist_t *plist; /* property list pointer */ + H5D_append_flush_t info; + unsigned u; /* local index variable */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "iIu*h*x**x", plist_id, ndims, boundary, func, udata); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Retrieve info for append flush */ + if(H5P_get(plist, H5D_ACS_APPEND_FLUSH_NAME, &info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get object flush callback") + + /* Assign return values */ + if(boundary) { + HDmemset(boundary, 0, ndims * sizeof(hsize_t)); + if(info.ndims > 0) + for(u = 0; u < info.ndims && u < ndims; u++) + boundary[u] = info.boundary[u]; + } /* end if */ + if(func) + *func = info.func; + if(udata) + *udata = info.udata; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_append_flush() */ + + +/*------------------------------------------------------------------------- * Function: H5Pset_efile_prefix * * Purpose: Set a prefix to be used for any external files. @@ -1165,3 +1293,4 @@ H5Pget_efile_prefix(hid_t plist_id, char *prefix, size_t size) done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_efile_prefix() */ + diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 9353094..e685c65 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -273,7 +273,7 @@ static const void *H5D_def_vlen_alloc_info_g = H5D_XFER_VLEN_ALLOC_INFO_DEF; / static const H5MM_free_t H5D_def_vlen_free_g = H5D_XFER_VLEN_FREE_DEF; /* Default value for vlen free function */ static const void *H5D_def_vlen_free_info_g = H5D_XFER_VLEN_FREE_INFO_DEF; /* Default value for vlen free information */ static const size_t H5D_def_hyp_vec_size_g = H5D_XFER_HYPER_VECTOR_SIZE_DEF; /* Default value for vector size */ -static const haddr_t H5D_def_metadata_tag_g = H5AC_METADATA_TAG_DEF; /* Default value for metadata tag */ +static const H5C_tag_t H5D_def_tag_g = H5C_TAG_DEF; /* Default value for cache entry tag */ static const H5FD_mpio_xfer_t H5D_def_io_xfer_mode_g = H5D_XFER_IO_XFER_MODE_DEF; /* Default value for I/O transfer mode */ static const H5FD_mpio_chunk_opt_t H5D_def_mpio_chunk_opt_mode_g = H5D_XFER_MPIO_CHUNK_OPT_HARD_DEF; static const H5FD_mpio_collective_opt_t H5D_def_mpio_collective_opt_mode_g = H5D_XFER_MPIO_COLLECTIVE_OPT_DEF; @@ -325,9 +325,8 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - /* Register the metadata tag property */ - /* (Note: this property should not have an encode/decode callback -QAK) */ - if(H5P_register_real(pclass, H5AC_METADATA_TAG_NAME, H5AC_METADATA_TAG_SIZE, &H5D_def_metadata_tag_g, + /* Register the cache tag property */ + if(H5P_register_real(pclass, H5C_TAG_NAME, H5C_TAG_SIZE, &H5D_def_tag_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 1fbc3b8..a315f92 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -178,6 +178,9 @@ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF 524288 #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC H5P__encode_size_t #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC H5P__decode_size_t +/* Definition for object flush callback */ +#define H5F_ACS_OBJECT_FLUSH_CB_SIZE sizeof(H5F_object_flush_t) +#define H5F_ACS_OBJECT_FLUSH_CB_DEF {NULL, NULL} #ifdef H5_HAVE_PARALLEL /* Definition of collective metadata read mode flag */ #define H5F_ACS_COLL_MD_READ_FLAG_SIZE sizeof(H5P_coll_md_read_flag_t) @@ -292,6 +295,7 @@ static const unsigned H5F_def_efc_size_g = H5F_ACS_EFC_SIZE_DEF; static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */ static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */ static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */ +static const H5F_object_flush_t H5F_def_object_flush_cb_g = H5F_ACS_OBJECT_FLUSH_CB_DEF; /* Default setting for object flush callback */ #ifdef H5_HAVE_PARALLEL static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_MD_READ_FLAG_DEF; /* Default setting for the collective metedata read flag */ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ @@ -452,6 +456,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register object flush callback */ + /* (Note: this property should not have an encode/decode callback -QAK) */ + if(H5P_register_real(pclass, H5F_ACS_OBJECT_FLUSH_CB_NAME, H5F_ACS_OBJECT_FLUSH_CB_SIZE, &H5F_def_object_flush_cb_g, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + #ifdef H5_HAVE_PARALLEL /* Register the metadata collective read flag */ if(H5P_register_real(pclass, H5_COLL_MD_READ_FLAG_NAME, H5F_ACS_COLL_MD_READ_FLAG_SIZE, &H5F_def_coll_md_read_flag_g, @@ -3510,6 +3520,91 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_core_write_tracking() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_obj_flush_cb + * + * Purpose: Sets the callback function to invoke and the user data when an + * object flush occurs in the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *udata) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5F_object_flush_t flush_info; + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ix*x", plist_id, func, udata); + + /* Check if the callback function is NULL and the user data is non-NULL. + * This is almost certainly an error as the user data will not be used. */ + if(!func && udata) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "callback is NULL while user data is not") + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Update property list */ + flush_info.func = func; + flush_info.udata = udata; + + /* Set values */ + if(H5P_set(plist, H5F_ACS_OBJECT_FLUSH_CB_NAME, &flush_info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set object flush callback") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_obj_flush_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_obj_flush_cb + * + * Purpose: Retrieves the callback function and user data set in the + * property list for an object flush. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_object_flush_cb(hid_t plist_id, H5F_flush_cb_t *func, void **udata) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5F_object_flush_t flush_info; + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*x**x", plist_id, func, udata); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Retrieve the callback function and user data */ + if(H5P_get(plist, H5F_ACS_OBJECT_FLUSH_CB_NAME, &flush_info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get object flush callback") + + /* Assign return value */ + if(func) + *func = flush_info.func; + if(udata) + *udata = flush_info.udata; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_obj_flush_cb() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 67727ce..0318c8f 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -351,6 +351,8 @@ H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callbacks_ptr); H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size); H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size); +H5_DLL herr_t H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *udata); +H5_DLL herr_t H5Pget_object_flush_cb(hid_t plist_id, H5F_flush_cb_t *func, void **udata); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); @@ -408,6 +410,10 @@ H5_DLL herr_t H5Pset_virtual_view(hid_t plist_id, H5D_vds_view_t view); H5_DLL herr_t H5Pget_virtual_view(hid_t plist_id, H5D_vds_view_t *view); H5_DLL herr_t H5Pset_virtual_printf_gap(hid_t plist_id, hsize_t gap_size); H5_DLL herr_t H5Pget_virtual_printf_gap(hid_t plist_id, hsize_t *gap_size); +H5_DLL herr_t H5Pset_append_flush(hid_t plist_id, unsigned ndims, + const hsize_t boundary[], H5D_append_cb_t func, void *udata); +H5_DLL herr_t H5Pget_append_flush(hid_t plist_id, unsigned dims, + hsize_t boundary[], H5D_append_cb_t *func, void **udata); H5_DLL herr_t H5Pset_efile_prefix(hid_t dapl_id, const char* prefix); H5_DLL ssize_t H5Pget_efile_prefix(hid_t dapl_id, char* prefix /*out*/, size_t size); diff --git a/src/Makefile.am b/src/Makefile.am index 5d72fc5..81fec77 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -81,7 +81,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5O.c H5Oainfo.c H5Oalloc.c H5Oattr.c \ H5Oattribute.c H5Obogus.c H5Obtreek.c H5Ocache.c H5Ochunk.c \ H5Ocont.c H5Ocopy.c H5Odbg.c H5Odrvinfo.c H5Odtype.c H5Oefl.c \ - H5Ofill.c H5Ofsinfo.c H5Oginfo.c \ + H5Ofill.c H5Oflush.c H5Ofsinfo.c H5Oginfo.c \ H5Olayout.c \ H5Olinfo.c H5Olink.c H5Omessage.c H5Omtime.c \ H5Oname.c H5Onull.c H5Opline.c H5Orefcount.c \ |