From 09b1106e39edc1d2fa7c92c9c8de57d8c01e4e71 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 30 Apr 2018 07:52:07 -0700 Subject: First stage of moving H5DOread/write_chunk() to src/ and making them H5D calls. * Moved H5DOread/write_chunk() to H5Dio.c and renamed to H5D*. * Moved the hl/test/test_dset_opt test to test/ and renamed to direct_chunk. * Moved the hl/test/dectris_hl_perf test to tools/test/perform and renamed to direct_write_perf. * Updated autotools and CMake files. --- hl/src/H5DO.c | 227 +--- hl/src/H5DOpublic.h | 9 - hl/test/CMakeLists.txt | 1 - hl/test/CMakeTests.cmake | 2 - hl/test/Makefile.am | 5 +- hl/test/dectris_hl_perf.c | 691 ---------- hl/test/test_dset_opt.c | 2171 -------------------------------- src/H5Dio.c | 155 ++- src/H5Dpublic.h | 4 + test/CMakeLists.txt | 1 + test/CMakeTests.cmake | 1 + test/Makefile.am | 13 +- test/direct_chunk.c | 2167 +++++++++++++++++++++++++++++++ tools/test/perform/direct_write_perf.c | 684 ++++++++++ 14 files changed, 3058 insertions(+), 3073 deletions(-) delete mode 100644 hl/test/dectris_hl_perf.c delete mode 100644 hl/test/test_dset_opt.c create mode 100644 test/direct_chunk.c create mode 100644 tools/test/perform/direct_write_perf.c diff --git a/hl/src/H5DO.c b/hl/src/H5DO.c index 99cf2f7..20d5a4a 100644 --- a/hl/src/H5DO.c +++ b/hl/src/H5DO.c @@ -22,163 +22,20 @@ /* public LT prototypes */ #include "H5DOpublic.h" - /*------------------------------------------------------------------------- - * Function: H5DOwrite_chunk - * - * Purpose: Writes an entire chunk to the file directly. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Raymond Lu - * 30 July 2012 + * Function: H5DOappend() * - *------------------------------------------------------------------------- - */ -herr_t -H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, - size_t data_size, const void *buf) -{ - hbool_t created_dxpl = FALSE; /* Whether we created a DXPL */ - hbool_t do_direct_write = TRUE; /* Flag for direct writes */ - uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ - herr_t ret_value = FAIL; /* Return value */ - - /* Check arguments */ - if(dset_id < 0) - goto done; - if(!buf) - goto done; - if(!offset) - goto done; - if(!data_size) - goto done; - data_size_32 = (uint32_t)data_size; - if(data_size != (size_t)data_size_32) - goto done; - - /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ - if(H5P_DEFAULT == dxpl_id) { - if((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto done; - created_dxpl = TRUE; - } /* end if */ - - /* Set direct write parameters */ - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) - goto done; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, &filters) < 0) - goto done; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, &offset) < 0) - goto done; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, &data_size_32) < 0) - goto done; - - /* Write chunk */ - if(H5Dwrite(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) - goto done; - - /* Indicate success */ - ret_value = SUCCEED; - -done: - if(created_dxpl) { - if(H5Pclose(dxpl_id) < 0) - ret_value = FAIL; - } /* end if */ - else { - /* Reset the direct write flag on user DXPL */ - do_direct_write = FALSE; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) - ret_value = FAIL; - } - - return ret_value; -} /* end H5DOwrite_chunk() */ - - -/*------------------------------------------------------------------------- - * Function: H5DOread_chunk + * Purpose: To append elements to a dataset. * - * Purpose: Reads an entire chunk from the file directly. + * axis: the dataset dimension (zero-based) for the append + * extension: the # of elements to append for the axis-th dimension + * memtype: the datatype + * buf: buffer with data for the append * * Return: Non-negative on success/Negative on failure * - * Programmer: Matthew Strong (GE Healthcare) - * 14 February 2016 - * - *--------------------------------------------------------------------------- - */ -herr_t -H5DOread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *filters, - void *buf) -{ - hbool_t created_dxpl = FALSE; /* Whether we created a DXPL */ - hbool_t do_direct_read = TRUE; /* Flag for direct writes */ - herr_t ret_value = FAIL; /* Return value */ - - /* Check arguments */ - if(dset_id < 0) - goto done; - if(!buf) - goto done; - if(!offset) - goto done; - if(!filters) - goto done; - - /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ - if(H5P_DEFAULT == dxpl_id) { - if((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto done; - created_dxpl = TRUE; - } /* end if */ - - /* Set direct write parameters */ - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, &do_direct_read) < 0) - goto done; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME, &offset) < 0) - goto done; - - /* Read chunk */ - if(H5Dread(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) - goto done; - /* Get the filter mask */ - if(H5Pget(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME, filters) < 0) - goto done; - - /* Indicate success */ - ret_value = SUCCEED; - -done: - if(created_dxpl) { - if(H5Pclose(dxpl_id) < 0) - ret_value = FAIL; - } /* end if */ - else { - /* Reset the direct read flag on user DXPL */ - do_direct_read = FALSE; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, &do_direct_read) < 0) - ret_value = FAIL; - } - - return ret_value; -} /* end H5DOread_chunk() */ - - -/*------------------------------------------------------------------------- - * Function: H5DOappend() - * - * Purpose: To append elements to a dataset. - * axis: the dataset dimension (zero-based) for the append - * extension: the # of elements to append for the axis-th dimension - * memtype: the datatype - * buf: buffer with data for the append - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Vailin Choi; Jan 2014 + * Programmer: Vailin Choi; Jan 2014 * * Note: * This routine is copied from the fast forward feature branch: features/hdf5_ff @@ -227,7 +84,7 @@ H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, /* check arguments */ if(H5I_DATASET != H5Iget_type(dset_id)) - goto done; + goto done; /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ if(H5P_DEFAULT == dxpl_id) { @@ -236,35 +93,35 @@ H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, created_dxpl = TRUE; } /* end if */ else if(TRUE != H5Pisa_class(dxpl_id, H5P_DATASET_XFER)) - goto done; + goto done; /* Get the dataspace of the dataset */ if(FAIL == (space_id = H5Dget_space(dset_id))) - goto done; + goto done; /* Get the rank of this dataspace */ if((sndims = H5Sget_simple_extent_ndims(space_id)) < 0) - goto done; + goto done; ndims = (unsigned)sndims; /* Verify correct axis */ if(axis >= ndims) - goto done; + goto done; /* Get the dimensions sizes of the dataspace */ if(H5Sget_simple_extent_dims(space_id, size, NULL) < 0) - goto done; + goto done; /* Adjust the dimension size of the requested dimension, but first record the old dimension size */ old_size = size[axis]; size[axis] += extension; if(size[axis] < old_size) - goto done; + goto done; /* Set the extent of the dataset to the new dimension */ if(H5Dset_extent(dset_id, size) < 0) - goto done; + goto done; /* Get the new dataspace of the dataset */ if(FAIL == (new_space_id = H5Dget_space(dset_id))) @@ -282,51 +139,51 @@ H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, } /* end if */ } /* end for */ if(FAIL == H5Sselect_hyperslab(new_space_id, H5S_SELECT_SET, start, stride, count, block)) - goto done; + goto done; /* The # of elemnts in the new extended dataspace */ if((snelmts = H5Sget_select_npoints(new_space_id)) < 0) - goto done; + goto done; nelmts = (hsize_t)snelmts; /* create a memory space */ if(FAIL == (mem_space_id = H5Screate_simple(1, &nelmts, NULL))) - goto done; + goto done; /* Write the data */ if(H5Dwrite(dset_id, memtype, mem_space_id, new_space_id, dxpl_id, buf) < 0) - goto done; + goto done; /* Obtain the dataset's access property list */ if((dapl = H5Dget_access_plist(dset_id)) < 0) - goto done; + goto done; /* Allocate the boundary array */ boundary = (hsize_t *)HDmalloc(ndims * sizeof(hsize_t)); /* Retrieve the append flush property */ if(H5Pget_append_flush(dapl, ndims, boundary, &append_cb, &udata) < 0) - goto done; + goto done; /* No boundary for this axis */ if(boundary[axis] != 0) { - /* Determine whether a boundary is hit or not */ - for(k = start[axis]; k < size[axis]; k++) - if(!((k + 1) % boundary[axis])) { - hit = TRUE; - break; - } - - if(hit) { /* Hit the boundary */ - /* Invoke callback if there is one */ - if(append_cb && append_cb(dset_id, size, udata) < 0) - goto done; - - /* Do a dataset flush */ - if(H5Dflush(dset_id) < 0) - goto done; - } /* end if */ + /* Determine whether a boundary is hit or not */ + for(k = start[axis]; k < size[axis]; k++) + if(!((k + 1) % boundary[axis])) { + hit = TRUE; + break; + } + + if(hit) { /* Hit the boundary */ + /* Invoke callback if there is one */ + if(append_cb && append_cb(dset_id, size, udata) < 0) + goto done; + + /* Do a dataset flush */ + if(H5Dflush(dset_id) < 0) + goto done; + } /* end if */ } /* end if */ /* Indicate success */ @@ -341,22 +198,22 @@ done: /* Close old dataspace */ if(space_id != FAIL && H5Sclose(space_id) < 0) - ret_value = FAIL; + ret_value = FAIL; /* Close new dataspace */ if(new_space_id != FAIL && H5Sclose(new_space_id) < 0) - ret_value = FAIL; + ret_value = FAIL; /* Close memory dataspace */ if(mem_space_id != FAIL && H5Sclose(mem_space_id) < 0) - ret_value = FAIL; + ret_value = FAIL; /* Close the dataset access property list */ if(dapl != FAIL && H5Pclose(dapl) < 0) - ret_value = FAIL; + ret_value = FAIL; if(boundary) - HDfree(boundary); + HDfree(boundary); return ret_value; } /* H5DOappend() */ diff --git a/hl/src/H5DOpublic.h b/hl/src/H5DOpublic.h index d5c8de4..7f83a7f 100644 --- a/hl/src/H5DOpublic.h +++ b/hl/src/H5DOpublic.h @@ -25,15 +25,6 @@ extern "C" { *------------------------------------------------------------------------- */ -H5_HLDLL herr_t H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, - const hsize_t *offset, size_t data_size, const void *buf); - -H5_HLDLL herr_t H5DOread_chunk(hid_t dset_id, /*in*/ - hid_t dxpl_id, /*in*/ - const hsize_t *offset, /*in*/ - uint32_t *filters, /*out*/ - void *buf); /*out*/ - H5_HLDLL herr_t H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, hid_t memtype, const void *buf); diff --git a/hl/test/CMakeLists.txt b/hl/test/CMakeLists.txt index 6b3c764..8a059a2 100644 --- a/hl/test/CMakeLists.txt +++ b/hl/test/CMakeLists.txt @@ -47,7 +47,6 @@ HL_ADD_EXE (test_image) HL_ADD_EXE (test_file_image) HL_ADD_EXE (test_table) HL_ADD_EXE (test_ds) -HL_ADD_EXE (test_dset_opt) HL_ADD_EXE (test_ld) HL_ADD_EXE (test_dset_append) diff --git a/hl/test/CMakeTests.cmake b/hl/test/CMakeTests.cmake index e5eb58e..6f57a8c 100644 --- a/hl/test/CMakeTests.cmake +++ b/hl/test/CMakeTests.cmake @@ -90,7 +90,6 @@ add_test ( test_ds8.h5 test_ds9.h5 test_ds10.h5 - test_dectris.h5 test_image1.h5 test_image2.h5 test_image3.h5 @@ -115,7 +114,6 @@ HL_add_test (test_file_image) HL_add_test (test_table) HL_add_test (test_ds) HL_add_test (test_packet) -HL_add_test (test_dset_opt) HL_add_test (test_ld) HL_add_test (test_dset_append) diff --git a/hl/test/Makefile.am b/hl/test/Makefile.am index e16550f..35e680c 100644 --- a/hl/test/Makefile.am +++ b/hl/test/Makefile.am @@ -26,7 +26,7 @@ LDADD=$(LIBH5_HL) $(LIBH5TEST) $(LIBHDF5) # Test programs. These are our main targets. They should be listed in the # order to be executed, generally most specific tests to least specific tests. -TEST_PROG=test_lite test_image test_file_image test_table test_ds test_packet test_dset_opt \ +TEST_PROG=test_lite test_image test_file_image test_table test_ds test_packet \ test_ld test_dset_append check_PROGRAMS=$(TEST_PROG) @@ -45,8 +45,7 @@ endif CHECK_CLEANFILES+=combine_tables[1-2].h5 test_ds[1-9].h5 test_ds10.h5 \ test_image[1-3].h5 file_img[1-2].h5 test_lite[1-4].h5 test_table.h5 \ test_packet_table.h5 test_packet_compress.h5 test_detach.h5 \ - test_packet_table_vlen.h5 testfl_packet_table_vlen.h5 \ - test_dectris.h5 test_append.h5 + test_packet_table_vlen.h5 testfl_packet_table_vlen.h5 test_append.h5 # Sources for test_packet executable test_packet_SOURCES=test_packet.c test_packet_vlen.c diff --git a/hl/test/dectris_hl_perf.c b/hl/test/dectris_hl_perf.c deleted file mode 100644 index 13cfac8..0000000 --- a/hl/test/dectris_hl_perf.c +++ /dev/null @@ -1,691 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * This test is for the DECTRIS project to the H5DOwrite_chunk function - * - */ - -#include "hdf5_hl.h" - -#ifdef H5_HAVE_FILTER_DEFLATE -#include - -#if !defined(WIN32) && !defined(__MINGW32__) - -#include - -#ifdef H5_STDC_HEADERS -# include -# include -# include -# include -#endif - -#ifdef H5_HAVE_UNISTD_H -# include -# include -#endif - -#ifdef H5_HAVE_SYS_STAT_H -# include -#endif - -#if defined(H5_TIME_WITH_SYS_TIME) -# include -# include -#elif defined(H5_HAVE_SYS_TIME_H) -# include -#else -# include -#endif - -const char *FILENAME[] = { - "dectris_perf", - "unix.raw", - NULL -}; - -/* - * Print the current location on the standard output stream. - */ -#define FUNC __func__ -#define AT() printf (" at %s:%d in %s()...\n", \ - __FILE__, __LINE__, FUNC); -#define H5_FAILED() {puts("*FAILED*");fflush(stdout);} -#define TEST_ERROR {H5_FAILED(); AT(); goto error;} -#define TESTING(WHAT) {printf("Testing %-62s",WHAT); fflush(stdout);} -#define PASSED() {puts(" PASSED");fflush(stdout);} - -#define DIRECT_UNCOMPRESSED_DSET "direct_uncompressed_dset" -#define DIRECT_COMPRESSED_DSET "direct_compressed_dset" -#define REG_COMPRESSED_DSET "reg_compressed_dset" -#define REG_NO_COMPRESS_DSET "reg_no_compress_dset" -#define RANK 3 -#define NX 100 -#define NY 1000 -#define NZ 250 -#define CHUNK_NX 1 -#define CHUNK_NY 1000 -#define CHUNK_NZ 250 - -#define DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s))*1.001)+12) -char filename[1024]; -unsigned int *outbuf[NX]; -size_t data_size[NX]; -double total_size = 0.0; -unsigned int *direct_buf[NX]; -double MB = 1048576.0; - -/*-------------------------------------------------- - * Function to report IO rate - *-------------------------------------------------- - */ -void reportTime(struct timeval start, double mbytes) -{ - struct timeval timeval_stop,timeval_diff; - - /*end timing*/ - gettimeofday(&timeval_stop,NULL); - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec=timeval_stop.tv_usec-start.tv_usec; - timeval_diff.tv_sec=timeval_stop.tv_sec-start.tv_sec; - - if(timeval_diff.tv_usec<0) { - timeval_diff.tv_usec+=1000000; - timeval_diff.tv_sec--; - } /* end if */ - -/*printf("mbytes=%lf, sec=%lf, usec=%lf\n", mbytes, (double)timeval_diff.tv_sec, (double)timeval_diff.tv_usec);*/ - printf("MBytes/second: %lf\n", (double)mbytes/((double)timeval_diff.tv_sec+((double)timeval_diff.tv_usec/(double)1000000.0))); -} - -/*-------------------------------------------------- - * Create file, datasets, and initialize data - *-------------------------------------------------- - */ -int create_file(hid_t fapl_id) -{ - hid_t file; /* handles */ - hid_t fapl; - hid_t cparms; - hid_t dataspace, dataset; - hsize_t dims[RANK] = {NX, NY, NZ}; - hsize_t chunk_dims[RANK] ={CHUNK_NX, CHUNK_NY, CHUNK_NZ}; - unsigned int aggression = 9; /* Compression aggression setting */ - int ret; - int i, j, n; - - int flag; - int unix_file; - - unsigned int *p; - size_t buf_size = CHUNK_NY*CHUNK_NZ*sizeof(unsigned int); - - const Bytef *z_src; - Bytef *z_dst; /*destination buffer */ - uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); - uLong z_src_nbytes = (uLong)buf_size; - - TESTING("Create a file and dataset"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) - TEST_ERROR; - - /* - * Create a new file. If file exists its contents will be overwritten. - */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) - TEST_ERROR; - - /* - * Modify dataset creation properties, i.e. enable chunking and compression - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - TEST_ERROR; - - if(H5Pset_chunk( cparms, RANK, chunk_dims) < 0) - TEST_ERROR; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DIRECT_UNCOMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - TEST_ERROR; - - if(H5Dclose(dataset) < 0) - TEST_ERROR; - - if((dataset = H5Dcreate2(file, REG_NO_COMPRESS_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - TEST_ERROR; - - if(H5Dclose(dataset) < 0) - TEST_ERROR; - - /* Set compression */ - if(H5Pset_deflate( cparms, aggression) < 0) - TEST_ERROR; - - if((dataset = H5Dcreate2(file, DIRECT_COMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - TEST_ERROR; - - if(H5Dclose(dataset) < 0) - TEST_ERROR; - - - if((dataset = H5Dcreate2(file, REG_COMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - TEST_ERROR; - - if(H5Dclose(dataset) < 0) - TEST_ERROR; - - if(H5Fclose(file) < 0) - TEST_ERROR; - - if(H5Sclose(dataspace) < 0) - TEST_ERROR; - - if(H5Pclose(cparms) < 0) - TEST_ERROR; - - /* create a unix file*/ - flag = O_CREAT|O_TRUNC|O_WRONLY; - - if ((unix_file=open(FILENAME[1],flag,S_IRWXU))== -1) - TEST_ERROR; - - if (close(unix_file) < 0) - { - printf(" unable to close the file\n"); - TEST_ERROR; - } - - - /* Initialize data for chunks */ - for(i = 0; i < NX; i++) { - p = direct_buf[i] = (unsigned int*)malloc(CHUNK_NY*CHUNK_NZ*sizeof(unsigned int)); - - for(j=0; j < CHUNK_NY*CHUNK_NZ; j++, p++) - *p = rand() % 65000; - - z_src = (const Bytef*)direct_buf[i]; - - z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); - /* Allocate output (compressed) buffer */ - outbuf[i] = (unsigned int*)malloc((size_t)z_dst_nbytes); - z_dst = (Bytef *)outbuf[i]; - - /* Perform compression from the source to the destination buffer */ - ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); - - data_size[i] = (size_t)z_dst_nbytes; - total_size += data_size[i]; - - /* Check for various zlib errors */ - if(Z_BUF_ERROR == ret) { - fprintf(stderr, "overflow"); - TEST_ERROR; - } else if(Z_MEM_ERROR == ret) { - fprintf(stderr, "deflate memory error"); - TEST_ERROR; - } else if(Z_OK != ret) { - fprintf(stderr, "other deflate error"); - TEST_ERROR; - } - } - - - PASSED(); - -error: - H5E_BEGIN_TRY { - H5Dclose(dataset); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Fclose(file); - } H5E_END_TRY; - return 1; -} - -/*-------------------------------------------------- - * Benchmark the performance of the new function - * with precompressed data. - *-------------------------------------------------- - */ -int -test_direct_write_uncompressed_data(hid_t fapl_id) -{ - hid_t file; /* handles */ - hid_t dataspace, dataset; - hid_t dxpl; - herr_t status; - int i; - - unsigned filter_mask = 0; - hsize_t offset[RANK] = {0, 0, 0}; - - struct timeval timeval_start; - - TESTING("H5DOwrite_chunk for uncompressed data"); - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - TEST_ERROR; - - /* Start the timer */ - gettimeofday(&timeval_start,NULL); - - /* Reopen the file and dataset */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) - TEST_ERROR; - - if((dataset = H5Dopen2(file, DIRECT_UNCOMPRESSED_DSET, H5P_DEFAULT)) < 0) - TEST_ERROR; - - - /* Write the compressed chunk data repeatedly to cover all the chunks in the - * dataset, using the direct writing function. */ - for(i=0; i -#include -#include "h5hltest.h" -#include "H5DOpublic.h" -#include - -#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) -# define H5_ZLIB_HEADER "zlib.h" -#endif -#if defined(H5_ZLIB_HEADER) -# include H5_ZLIB_HEADER /* "zlib.h" */ -#endif - -#define FILE_NAME "test_dectris.h5" - -/* Datasets for Direct Write tests */ -#define DATASETNAME1 "direct_write" -#define DATASETNAME2 "skip_one_filter" -#define DATASETNAME3 "skip_two_filters" -#define DATASETNAME4 "data_conv" -#define DATASETNAME5 "contiguous_dset" -#define DATASETNAME6 "invalid_argue" -#define DATASETNAME7 "overwrite_chunk" -/* Datasets for Direct Read tests */ -#define DATASETNAME8 "disabled_chunk_cache" -#define DATASETNAME9 "flush_chunk_cache" -#define DATASETNAME10 "read_w_valid_cache" -#define DATASETNAME11 "unallocated_chunk" -#define DATASETNAME12 "unfiltered_data" - -#define RANK 2 -#define NX 16 -#define NY 16 -#define CHUNK_NX 4 -#define CHUNK_NY 4 - -#define DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*H5_DOUBLE(1.001))+H5_DOUBLE(12.0)) - -/* Temporary filter IDs used for testing */ -#define H5Z_FILTER_BOGUS1 305 -#define H5Z_FILTER_BOGUS2 306 -#define ADD_ON 7 -#define FACTOR 3 - -/* Constants for the overwrite test */ -#define OVERWRITE_NDIMS 3 -#define OVERWRITE_CHUNK_NX 3 -#define OVERWRITE_CHUNK_2NX 6 -#define OVERWRITE_CHUNK_NY 2 -#define OVERWRITE_VALUE 42 - -/* Defines used in test_single_chunk_latest() */ -#define FILE "single_latest.h5" -#define DATASET "dataset" -#define DIM0 4 -#define DIM1 32 -#define CHUNK0 DIM0 -#define CHUNK1 DIM1 - -/* Local prototypes for filter functions */ -static size_t filter_bogus1(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); -static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); - -/* This message derives from H5Z */ -const H5Z_class2_t H5Z_BOGUS1[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - H5Z_FILTER_BOGUS1, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ - "bogus1", /* Filter name for debugging */ - NULL, /* The "can apply" callback */ - NULL, /* The "set local" callback */ - filter_bogus1, /* The actual filter function */ -}}; - -const H5Z_class2_t H5Z_BOGUS2[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - H5Z_FILTER_BOGUS2, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ - "bogus2", /* Filter name for debugging */ - NULL, /* The "can apply" callback */ - NULL, /* The "set local" callback */ - filter_bogus2, /* The actual filter function */ -}}; - -/*------------------------------------------------------------------------- - * Function: test_direct_chunk_write - * - * Purpose: Test the basic functionality of H5DOwrite_chunk - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Raymond Lu - * 30 November 2012 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_FILTER_DEFLATE -static int -test_direct_chunk_write (hid_t file) -{ - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int ret; - int data[NX][NY]; - int i, j, n; - - unsigned filter_mask = 0; - int direct_buf[CHUNK_NX][CHUNK_NY]; - int check_chunk[CHUNK_NX][CHUNK_NY]; - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - - const Bytef *z_src = (const Bytef*)(direct_buf); - Bytef *z_dst = NULL; /*destination buffer */ - uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); - uLong z_src_nbytes = (uLong)buf_size; - int aggression = 9; /* Compression aggression setting */ - void *outbuf = NULL; /* Pointer to new buffer */ - - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ - - TESTING("basic functionality of H5DOwrite_chunk"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties, i.e. enable chunking and compression - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - - if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME1, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - /* Initialize the dataset */ - for(i = n = 0; i < NX; i++) - for(j = 0; j < NY; j++) - data[i][j] = n++; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* - * Write the data for the dataset. It should stay in the chunk cache. - * It will be evicted from the cache by the H5DOwrite_chunk calls. - */ - if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - dxpl, data)) < 0) - goto error; - - /* Initialize data for one chunk */ - for(i = n = 0; i < CHUNK_NX; i++) - for(j = 0; j < CHUNK_NY; j++) - direct_buf[i][j] = n++; - - /* Allocate output (compressed) buffer */ - outbuf = HDmalloc(z_dst_nbytes); - z_dst = (Bytef *)outbuf; - - /* Perform compression from the source to the destination buffer */ - ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); - - /* Check for various zlib errors */ - if(Z_BUF_ERROR == ret) { - HDfprintf(stderr, "overflow"); - goto error; - } else if(Z_MEM_ERROR == ret) { - HDfprintf(stderr, "deflate memory error"); - goto error; - } else if(Z_OK != ret) { - HDfprintf(stderr, "other deflate error"); - goto error; - } - - /* Write the compressed chunk data repeatedly to cover all the chunks in the - * dataset, using the direct writing function. */ - for(i=0; i0) { - *int_ptr++ -= (int)ADD_ON; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end if */ - else { /* write */ - /* Add the "add on" value to all the data values */ - while(buf_left>0) { - *int_ptr++ += (int)ADD_ON; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end else */ - - return nbytes; -} /* filter_bogus1() */ - -/*------------------------------------------------------------------------- - * Function: filter_bogus2 - * - * Purpose: A bogus filter that multiplies the original value by FACTOR. - * - * Return: Success: Data chunk size - * - * Programmer: Raymond Lu - * 30 November 2012 - *------------------------------------------------------------------------- - */ -static size_t -filter_bogus2(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, - const unsigned int H5_ATTR_UNUSED *cd_values, size_t nbytes, - size_t *buf_size, void **buf) -{ - int *int_ptr=(int *)*buf; /* Pointer to the data values */ - ssize_t buf_left=(ssize_t)*buf_size; /* Amount of data buffer left to process */ - - if(flags & H5Z_FLAG_REVERSE) { /* read */ - /* Substract the "add on" value to all the data values */ - while(buf_left>0) { - *int_ptr++ /= (int)FACTOR; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end if */ - else { /* write */ - /* Add the "add on" value to all the data values */ - while(buf_left>0) { - *int_ptr++ *= (int)FACTOR; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end else */ - - return nbytes; -} /* filter_bogus2() */ - -/*------------------------------------------------------------------------- - * Function: test_skip_compress_write2 - * - * Purpose: Test skipping compression filter when there are three filters - * for the dataset - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Raymond Lu - * 30 November 2012 - * - *------------------------------------------------------------------------- - */ -static int -test_skip_compress_write2(hid_t file) -{ - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int i, j, n; - - unsigned filter_mask = 0; /* orig filter mask */ - int origin_direct_buf[CHUNK_NX][CHUNK_NY]; - int direct_buf[CHUNK_NX][CHUNK_NY]; - int check_chunk[CHUNK_NX][CHUNK_NY]; - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - int aggression = 9; /* Compression aggression setting */ - - unsigned read_filter_mask = 0; /* filter mask after direct read */ - int read_direct_buf[CHUNK_NX][CHUNK_NY]; - hsize_t read_buf_size = 0; /* buf size */ - - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ - - TESTING("skipping compression filters but keep two other filters"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties, i.e. enable chunking and compression. - * The order of filters is bogus 1 + deflate + bogus 2. - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - - /* Register and enable first bogus filter */ - if(H5Zregister (H5Z_BOGUS1) < 0) - goto error; - - if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS1, 0, (size_t)0, NULL) < 0) - goto error; - - /* Enable compression filter */ - if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) - goto error; - - /* Register and enable second bogus filter */ - if(H5Zregister (H5Z_BOGUS2) < 0) - goto error; - - if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS2, 0, (size_t)0, NULL) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME3, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Initialize data for one chunk. Apply operations of two bogus filters to the chunk */ - for(i = n = 0; i < CHUNK_NX; i++) - for(j = 0; j < CHUNK_NY; j++) { - origin_direct_buf[i][j] = n++; - direct_buf[i][j] = (origin_direct_buf[i][j] + ADD_ON) * FACTOR; - } - - /* write the uncompressed chunk data repeatedly to dataset, using the direct writing function. - * Indicate skipping the compression filter but keep the other two bogus filters */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - - /* compression filter is the middle one to be skipped */ - filter_mask = 0x00000002; - - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) - goto error; - - if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) - goto error; - - if(H5Dclose(dataset) < 0) - goto error; - - if((dataset = H5Dopen2(file, DATASETNAME3, H5P_DEFAULT)) < 0) - goto error; - - /* - * Select hyperslab for one chunk in the file - */ - start[0] = CHUNK_NX; start[1] = CHUNK_NY; - stride[0] = 1; stride[1] = 1; - count[0] = 1; count[1] = 1; - block[0] = CHUNK_NX; block[1] = CHUNK_NY; - if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) - goto error; - - /* Read the chunk back */ - if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) - goto error; - - /* Check that the values read are the same as the values written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if(origin_direct_buf[i][j] != check_chunk[i][j]) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" origin_direct_buf=%d, check_chunk=%d\n", origin_direct_buf[i][j], check_chunk[i][j]); - goto error; - } - } - } - - /* Query chunk storage size */ - if((status = H5Dget_chunk_storage_size(dataset, offset, &read_buf_size)) < 0) - goto error; - if(read_buf_size != buf_size) - goto error; - - /* Read the raw chunk back */ - HDmemset(&read_direct_buf, 0, sizeof(read_direct_buf)); - if((status = H5DOread_chunk(dataset, H5P_DEFAULT, offset, &read_filter_mask, read_direct_buf)) < 0) - goto error; - if(read_filter_mask != filter_mask) - goto error; - - /* Check that the direct chunk read is the same as the chunk written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if(direct_buf[i][j] != read_direct_buf[i][j]) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" direct_buf=%d, read_direct_buf=%d\n", direct_buf[i][j], read_direct_buf[i][j]); - goto error; - } - } - } - - /* - * Close/release resources. - */ - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - - PASSED(); - return 0; - -error: - H5E_BEGIN_TRY { - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - } H5E_END_TRY; - - H5_FAILED(); - return 1; -} /* test_skip_compress_write2() */ - -/*------------------------------------------------------------------------- - * Function: test_data_conv - * - * Purpose: Test data conversion - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Raymond Lu - * 30 November 2012 - * - *------------------------------------------------------------------------- - */ -static int -test_data_conv(hid_t file) -{ - typedef struct { - int a, b, c[4], d, e; - } src_type_t; - typedef struct { - int a, c[4], e; - } dst_type_t; - - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int i, j, n; - const hsize_t four = 4; - hid_t st=-1, dt=-1; - hid_t array_dt; - - unsigned filter_mask = 0; - src_type_t direct_buf[CHUNK_NX][CHUNK_NY]; - dst_type_t check_chunk[CHUNK_NX][CHUNK_NY]; - src_type_t read_chunk[CHUNK_NX][CHUNK_NY]; /* For H5DOread_chunk */ - - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(src_type_t); - - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ - - TESTING("data conversion for H5DOwrite_chunk/H5DOread_chunk"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties, i.e. enable chunking - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - - /* Build hdf5 datatypes */ - array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); - if((st = H5Tcreate(H5T_COMPOUND, sizeof(src_type_t))) < 0 || - H5Tinsert(st, "a", HOFFSET(src_type_t, a), H5T_NATIVE_INT) < 0 || - H5Tinsert(st, "b", HOFFSET(src_type_t, b), H5T_NATIVE_INT) < 0 || - H5Tinsert(st, "c", HOFFSET(src_type_t, c), array_dt) < 0 || - H5Tinsert(st, "d", HOFFSET(src_type_t, d), H5T_NATIVE_INT) < 0 || - H5Tinsert(st, "e", HOFFSET(src_type_t, e), H5T_NATIVE_INT) < 0) - goto error; - - if(H5Tclose(array_dt) < 0) - goto error; - - array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); - if((dt = H5Tcreate(H5T_COMPOUND, sizeof(dst_type_t))) < 0 || - H5Tinsert(dt, "a", HOFFSET(dst_type_t, a), H5T_NATIVE_INT) < 0 || - H5Tinsert(dt, "c", HOFFSET(dst_type_t, c), array_dt) < 0 || - H5Tinsert(dt, "e", HOFFSET(dst_type_t, e), H5T_NATIVE_INT) < 0) - goto error; - - if(H5Tclose(array_dt) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME4, st, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Initialize data for one chunk */ - for(i = n = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - (direct_buf[i][j]).a = i*j+0; - (direct_buf[i][j]).b = i*j+1; - (direct_buf[i][j]).c[0] = i*j+2; - (direct_buf[i][j]).c[1] = i*j+3; - (direct_buf[i][j]).c[2] = i*j+4; - (direct_buf[i][j]).c[3] = i*j+5; - (direct_buf[i][j]).d = i*j+6; - (direct_buf[i][j]).e = i*j+7; - } - } - - /* write the chunk data to dataset, using the direct writing function. - * There should be no data conversion involved. */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) - goto error; - - if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) - goto error; - - if(H5Dclose(dataset) < 0) - goto error; - - if((dataset = H5Dopen2(file, DATASETNAME4, H5P_DEFAULT)) < 0) - goto error; - - /* Use H5DOread_chunk() to read the uncompressed data */ - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, read_chunk)) < 0) - goto error; - - /* Check that the values read are the same as the values written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if ((direct_buf[i][j]).a != (read_chunk[i][j]).a || - (direct_buf[i][j]).b != (read_chunk[i][j]).b || - (direct_buf[i][j]).c[0] != (read_chunk[i][j]).c[0] || - (direct_buf[i][j]).c[1] != (read_chunk[i][j]).c[1] || - (direct_buf[i][j]).c[2] != (read_chunk[i][j]).c[2] || - (direct_buf[i][j]).c[3] != (read_chunk[i][j]).c[3] || - (direct_buf[i][j]).d != (read_chunk[i][j]).d || - (direct_buf[i][j]).e != (read_chunk[i][j]).e) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], - (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); - HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - (read_chunk[i][j]).a, (read_chunk[i][j]).b, (read_chunk[i][j]).c[0], (read_chunk[i][j]).c[1], - (read_chunk[i][j]).c[2], (read_chunk[i][j]).c[3], (read_chunk[i][j]).d, (read_chunk[i][j]).e); - - goto error; - } - } - } - - /* - * Select hyperslab for the chunk just written in the file - */ - start[0] = CHUNK_NX; start[1] = CHUNK_NY; - stride[0] = 1; stride[1] = 1; - count[0] = 1; count[1] = 1; - block[0] = CHUNK_NX; block[1] = CHUNK_NY; - if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) - goto error; - - /* Read the chunk back. Data should be converted */ - if((status = H5Dread(dataset, dt, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) - goto error; - - /* Check that the values read are the same as the values written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if ((direct_buf[i][j]).a != (check_chunk[i][j]).a || - (direct_buf[i][j]).c[0] != (check_chunk[i][j]).c[0] || - (direct_buf[i][j]).c[1] != (check_chunk[i][j]).c[1] || - (direct_buf[i][j]).c[2] != (check_chunk[i][j]).c[2] || - (direct_buf[i][j]).c[3] != (check_chunk[i][j]).c[3] || - (direct_buf[i][j]).e != (check_chunk[i][j]).e) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], - (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); - HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n", - (check_chunk[i][j]).a, (check_chunk[i][j]).c[0], (check_chunk[i][j]).c[1], (check_chunk[i][j]).c[2], - (check_chunk[i][j]).c[3], (check_chunk[i][j]).e); - - goto error; - } - } - } - - /* - * Close/release resources. - */ - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - H5Tclose(st); - H5Tclose(dt); - - PASSED(); - return 0; - -error: - H5E_BEGIN_TRY { - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - H5Tclose(st); - H5Tclose(dt); - } H5E_END_TRY; - - H5_FAILED(); - return 1; -} /* test_data_conv() */ - -/*------------------------------------------------------------------------- - * Function: test_invalid_parameters - * - * Purpose: Test invalid parameters for H5DOwrite_chunk and H5DOread_chunk - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Raymond Lu - * 30 November 2012 - * - *------------------------------------------------------------------------- - */ -static int -test_invalid_parameters(hid_t file) -{ - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int i, j, n; - - unsigned filter_mask = 0; - int direct_buf[CHUNK_NX][CHUNK_NY]; - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - int aggression = 9; /* Compression aggression setting */ - - hsize_t chunk_nbytes; /* Chunk size */ - - TESTING("invalid parameters for H5DOwrite_chunk/H5DOread_chunk"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - - /* - * Create a new contiguous dataset to verify H5DOwrite_chunk/H5DOread_chunk doesn't work - */ - if((dataset = H5Dcreate2(file, DATASETNAME5, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Initialize data for one chunk */ - for(i = n = 0; i < CHUNK_NX; i++) - for(j = 0; j < CHUNK_NY; j++) { - direct_buf[i][j] = n++; - } - - /* Try to write the chunk data to contiguous dataset. It should fail */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Try to get chunk size for a contiguous dataset. It should fail */ - H5E_BEGIN_TRY { - if((status = H5Dget_chunk_storage_size(dataset, offset, &chunk_nbytes)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Try to H5DOread_chunk from the contiguous dataset. It should fail */ - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - if(H5Dclose(dataset) < 0) - goto error; - - - /* Create a chunked dataset with compression filter */ - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - - if((status = H5Pset_deflate( cparms, (unsigned ) aggression)) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME6, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - /* Check invalid dataset ID for H5DOwrite_chunk and H5DOread_chunk */ - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk((hid_t)-1, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk((hid_t)-1, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid DXPL ID for H5DOwrite_chunk and H5DOread_chunk */ - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, (hid_t)-1, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, (hid_t)-1, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid OFFSET for H5DOwrite_chunk and H5DOread_chunk */ - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, NULL, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, NULL, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check when OFFSET is out of dataset range for H5DOwrite_chunk and H5DOread_chunk */ - offset[0] = NX + 1; - offset[1] = NY; - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check when OFFSET is not on chunk boundary for H5DOwrite_chunk and H5DOread_chunk */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY + 1; - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid buffer size for H5DOwrite_chunk only */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - buf_size = 0; - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid data buffer for H5DOwrite_chunk and H5DOread_chunk */ - buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, NULL)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, NULL)) != FAIL) - goto error; - } H5E_END_TRY; - - if(H5Dclose(dataset) < 0) - goto error; - - /* - * Close/release resources. - */ - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - - PASSED(); - return 0; - -error: - H5E_BEGIN_TRY { - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - } H5E_END_TRY; - - H5_FAILED(); - return 1; -} /* test_invalid_parameters() */ - -/*------------------------------------------------------------------------- - * Function: test_direct_chunk_read_no_cache - * - * Purpose: Test the basic functionality of H5DOread_chunk with the - * chunk cache diabled. - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Matthew Strong (GE Healthcare) - * 14 February 2016 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_FILTER_DEFLATE -static int -test_direct_chunk_read_no_cache (hid_t file) -{ - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1, dapl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; - herr_t status; /* status from H5 function calls */ - int ret; /* deflate return status */ - int data[NX][NY]; - int i, j, k, l, n; /* local index variables */ - - unsigned filter_mask = 0; /* filter mask returned from H5DOread_chunk */ - int direct_buf[CHUNK_NX][CHUNK_NY]; /* chunk read with H5DOread and manually decompressed */ - int check_chunk[CHUNK_NX][CHUNK_NY]; /* chunk read with H5Dread */ - hsize_t offset[2]; /* chunk offset used for H5DOread_chunk */ - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - - Bytef *z_src = NULL; /* source buffer */ - uLongf z_src_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); - Bytef *z_dst = (Bytef*)(direct_buf); - uLong z_dst_nbytes = (uLong)buf_size; - int aggression = 9; /* Compression aggression setting */ - void *outbuf = NULL; /* Pointer to new buffer */ - - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ - - TESTING("basic functionality of H5DOread_chunk (chunk cache disabled)"); - - /* Create the data space with unlimited dimensions. */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* Modify dataset creation properties, i.e. enable chunking and compression */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) - goto error; - if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) - goto error; - - /* Disable chunk cache by setting number of slots to 0 */ - if((status = H5Pset_chunk_cache(dapl, 0, H5D_CHUNK_CACHE_NBYTES_DEFAULT, H5D_CHUNK_CACHE_W0_DEFAULT)) < 0) - goto error; - - /* Create a new dataset within the file using cparms creation properties. */ - if((dataset = H5Dcreate2(file, DATASETNAME8, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, dapl)) < 0) - goto error; - - /* Initialize the dataset */ - for(i = n = 0; i < NX; i++) - for(j = 0; j < NY; j++) - data[i][j] = n++; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Write the data for the dataset. - * Data will skip chunk cache and go directly to disk. */ - if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - dxpl, data)) < 0) - goto error; - - /* Allocate output (compressed) buffer */ - outbuf = HDmalloc(z_src_nbytes); - z_src = (Bytef *)outbuf; - - /* For each chunk in the dataset, compare the result of H5Dread and H5DOread_chunk. */ - for(i=0; ioloc.file) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") - - if(mem_space_id < 0 || file_space_id < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file") + if(mem_space_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid mem_space_id") + if(file_space_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file_space_id") if(H5S_ALL != mem_space_id) { if(NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) @@ -215,6 +216,76 @@ done: /*------------------------------------------------------------------------- + * Function: H5Dread_chunk + * + * Purpose: Reads an entire chunk from the file directly. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Matthew Strong (GE Healthcare) + * 14 February 2016 + * + *--------------------------------------------------------------------------- + */ +herr_t +H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *filters, + void *buf) +{ + hbool_t created_dxpl = FALSE; /* Whether we created a DXPL */ + hbool_t do_direct_read = TRUE; /* Flag for direct writes */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "ii*h*Iu*x", dset_id, dxpl_id, offset, filters, buf); + + /* Check arguments */ + if(dset_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset ID") + if(!buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf cannot be NULL") + if(!offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL") + if(!filters) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filters cannot be NULL") + + /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ + if(H5P_DEFAULT == dxpl_id) { + if((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, FAIL, "cannot create dxpl") + created_dxpl = TRUE; + } /* end if */ + + /* Set direct write parameters */ + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, &do_direct_read) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set direct read property") + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME, &offset) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set offset property") + + /* Read chunk */ + if(H5Dread(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "cannot read from dataset") + + /* Get the filter mask */ + if(H5Pget(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME, filters) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "cannot get filter mask property") + +done: + if(created_dxpl) { + if(H5Pclose(dxpl_id) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTRELEASE, FAIL, "unable to close dxpl") + } /* end if */ + else { + /* Reset the direct read flag on user DXPL */ + do_direct_read = FALSE; + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, &do_direct_read) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot reset direct write property") + } + + FUNC_LEAVE_API(ret_value) +} /* end H5Dread_chunk() */ + + +/*------------------------------------------------------------------------- * Function: H5Dwrite * * Purpose: Writes (part of) a DSET from application memory BUF to the @@ -311,6 +382,80 @@ done: /*------------------------------------------------------------------------- + * Function: H5Dwrite_chunk + * + * Purpose: Writes an entire chunk to the file directly. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 30 July 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, + size_t data_size, const void *buf) +{ + hbool_t created_dxpl = FALSE; /* Whether we created a DXPL */ + hbool_t do_direct_write = TRUE; /* Flag for direct writes */ + uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "iiIu*hz*x", dset_id, dxpl_id, filters, offset, data_size, buf); + + /* Check arguments */ + if(dset_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset ID") + if(!buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf cannot be NULL") + if(!offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL") + if(!data_size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "data_size cannot be NULL") + data_size_32 = (uint32_t)data_size; + if(data_size != (size_t)data_size_32) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid data_size") + + /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ + if(H5P_DEFAULT == dxpl_id) { + if((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, FAIL, "cannot create dxpl") + created_dxpl = TRUE; + } /* end if */ + + /* Set direct write parameters */ + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set direct read property") + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, &filters) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set filters property") + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, &offset) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set offset property") + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, &data_size_32) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set data size property") + + /* Write chunk */ + if(H5Dwrite(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "cannot write to dataset") + +done: + if(created_dxpl) { + if(H5Pclose(dxpl_id) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTRELEASE, FAIL, "unable to close dxpl") + } /* end if */ + else { + /* Reset the direct write flag on user DXPL */ + do_direct_write = FALSE; + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot reset direct write property") + } + + FUNC_LEAVE_API(ret_value) +} /* end H5Dwrite_chunk() */ + + +/*------------------------------------------------------------------------- * Function: H5D__pre_write * * Purpose: Preparation for writing data. diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 790674a..be80cef 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -157,6 +157,10 @@ H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf/*out*/); H5_DLL herr_t H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf); +H5_DLL herr_t H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, + const hsize_t *offset, size_t data_size, const void *buf); +H5_DLL herr_t H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, + const hsize_t *offset, uint32_t *filters, void *buf); H5_DLL herr_t H5Diterate(void *buf, hid_t type_id, hid_t space_id, H5D_operator_t op, void *operator_data); H5_DLL herr_t H5Dvlen_reclaim(hid_t type_id, hid_t space_id, hid_t plist_id, void *buf); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ec560f2..0d93211 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -214,6 +214,7 @@ set (H5_TESTS cmpd_dset filter_fail extend + direct_chunk external efc objcopy diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index 04cc41f..6f914ba 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -532,6 +532,7 @@ set (test_CLEANFILES vds_swmr.h5 vds_swmr_src_*.h5 tmp/vds_src_2.h5 + direct_chunk.h5 ) # Remove any output file left over from previous test run diff --git a/test/Makefile.am b/test/Makefile.am index ab81f50..8f671ac 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -55,11 +55,12 @@ TEST_PROG= testhdf5 \ cache cache_api cache_image cache_tagging lheap ohdr stab gheap \ evict_on_close farray earray btree2 fheap \ pool accum hyperslab istore bittests dt_arith page_buffer \ - dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ - twriteorder big mtime fillval mount flush1 flush2 app_ref enum \ - set_extent ttsafe enc_dec_plist enc_dec_plist_cross_platform\ - getname vfd ntypes dangle dtransform reserved cross_read \ - freespace mf vds file_image unregister cache_logging cork swmr + dtypes dsets cmpd_dset filter_fail extend direct_chunk external efc \ + objcopy links unlink twriteorder big mtime fillval mount \ + flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ + enc_dec_plist_cross_platform getname vfd ntypes dangle dtransform \ + reserved cross_read freespace mf vds file_image unregister \ + cache_logging cork swmr # List programs to be built when testing here. # error_test and err_compat are built at the same time as the other tests, but executed by testerror.sh. @@ -190,7 +191,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse flushrefresh_VERIFICATION_DONE atomic_data accum_swmr_big.h5 ohdr_swmr.h5 \ test_swmr*.h5 cache_logging.h5 cache_logging.out vds_swmr.h5 vds_swmr_src_*.h5 \ swmr[0-2].h5 swmr_writer.out swmr_writer.log.* swmr_reader.out.* swmr_reader.log.* \ - tbogus.h5.copy cache_image_test.h5 + tbogus.h5.copy cache_image_test.h5 direct_chunk.h5 # Sources for testhdf5 executable testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ diff --git a/test/direct_chunk.c b/test/direct_chunk.c new file mode 100644 index 0000000..2ef38ea --- /dev/null +++ b/test/direct_chunk.c @@ -0,0 +1,2167 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Copyright by The HDF Group. * +* Copyright by the Board of Trustees of the University of Illinois. * +* All rights reserved. * +* * +* This file is part of HDF5. The full HDF5 copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5test.h" + +#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) +# define H5_ZLIB_HEADER "zlib.h" +#endif +#if defined(H5_ZLIB_HEADER) +# include H5_ZLIB_HEADER /* "zlib.h" */ +#endif + +#define FILE_NAME "direct_chunk.h5" + +/* Datasets for Direct Write tests */ +#define DATASETNAME1 "direct_write" +#define DATASETNAME2 "skip_one_filter" +#define DATASETNAME3 "skip_two_filters" +#define DATASETNAME4 "data_conv" +#define DATASETNAME5 "contiguous_dset" +#define DATASETNAME6 "invalid_argue" +#define DATASETNAME7 "overwrite_chunk" +/* Datasets for Direct Read tests */ +#define DATASETNAME8 "disabled_chunk_cache" +#define DATASETNAME9 "flush_chunk_cache" +#define DATASETNAME10 "read_w_valid_cache" +#define DATASETNAME11 "unallocated_chunk" +#define DATASETNAME12 "unfiltered_data" + +#define RANK 2 +#define NX 16 +#define NY 16 +#define CHUNK_NX 4 +#define CHUNK_NY 4 + +#define DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*H5_DOUBLE(1.001))+H5_DOUBLE(12.0)) + +/* Temporary filter IDs used for testing */ +#define H5Z_FILTER_BOGUS1 305 +#define H5Z_FILTER_BOGUS2 306 +#define ADD_ON 7 +#define FACTOR 3 + +/* Constants for the overwrite test */ +#define OVERWRITE_NDIMS 3 +#define OVERWRITE_CHUNK_NX 3 +#define OVERWRITE_CHUNK_2NX 6 +#define OVERWRITE_CHUNK_NY 2 +#define OVERWRITE_VALUE 42 + +/* Defines used in test_single_chunk_latest() */ +#define FILE "single_latest.h5" +#define DATASET "dataset" +#define DIM0 4 +#define DIM1 32 +#define CHUNK0 DIM0 +#define CHUNK1 DIM1 + +/* Local prototypes for filter functions */ +static size_t filter_bogus1(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_BOGUS1[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS1, /* Filter id number */ + 1, 1, /* Encoding and decoding enabled */ + "bogus1", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + filter_bogus1, /* The actual filter function */ +}}; + +const H5Z_class2_t H5Z_BOGUS2[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS2, /* Filter id number */ + 1, 1, /* Encoding and decoding enabled */ + "bogus2", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + filter_bogus2, /* The actual filter function */ +}}; + +/*------------------------------------------------------------------------- + * Function: test_direct_chunk_write + * + * Purpose: Test the basic functionality of H5Dwrite_chunk + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_FILTER_DEFLATE +static int +test_direct_chunk_write (hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int ret; + int data[NX][NY]; + int i, j, n; + + unsigned filter_mask = 0; + int direct_buf[CHUNK_NX][CHUNK_NY]; + int check_chunk[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + + const Bytef *z_src = (const Bytef*)(direct_buf); + Bytef *z_dst = NULL; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + uLong z_src_nbytes = (uLong)buf_size; + int aggression = 9; /* Compression aggression setting */ + void *outbuf = NULL; /* Pointer to new buffer */ + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("basic functionality of H5Dwrite_chunk"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME1, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + /* Initialize the dataset */ + for(i = n = 0; i < NX; i++) + for(j = 0; j < NY; j++) + data[i][j] = n++; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* + * Write the data for the dataset. It should stay in the chunk cache. + * It will be evicted from the cache by the H5Dwrite_chunk calls. + */ + if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + dxpl, data)) < 0) + goto error; + + /* Initialize data for one chunk */ + for(i = n = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) + direct_buf[i][j] = n++; + + /* Allocate output (compressed) buffer */ + outbuf = HDmalloc(z_dst_nbytes); + z_dst = (Bytef *)outbuf; + + /* Perform compression from the source to the destination buffer */ + ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == ret) { + HDfprintf(stderr, "overflow"); + goto error; + } else if(Z_MEM_ERROR == ret) { + HDfprintf(stderr, "deflate memory error"); + goto error; + } else if(Z_OK != ret) { + HDfprintf(stderr, "other deflate error"); + goto error; + } + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i0) { + *int_ptr++ -= (int)ADD_ON; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end if */ + else { /* write */ + /* Add the "add on" value to all the data values */ + while(buf_left>0) { + *int_ptr++ += (int)ADD_ON; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end else */ + + return nbytes; +} /* filter_bogus1() */ + +/*------------------------------------------------------------------------- + * Function: filter_bogus2 + * + * Purpose: A bogus filter that multiplies the original value by FACTOR. + * + * Return: Success: Data chunk size + * + * Programmer: Raymond Lu + * 30 November 2012 + *------------------------------------------------------------------------- + */ +static size_t +filter_bogus2(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, + const unsigned int H5_ATTR_UNUSED *cd_values, size_t nbytes, + size_t *buf_size, void **buf) +{ + int *int_ptr=(int *)*buf; /* Pointer to the data values */ + ssize_t buf_left=(ssize_t)*buf_size; /* Amount of data buffer left to process */ + + if(flags & H5Z_FLAG_REVERSE) { /* read */ + /* Substract the "add on" value to all the data values */ + while(buf_left>0) { + *int_ptr++ /= (int)FACTOR; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end if */ + else { /* write */ + /* Add the "add on" value to all the data values */ + while(buf_left>0) { + *int_ptr++ *= (int)FACTOR; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end else */ + + return nbytes; +} /* filter_bogus2() */ + +/*------------------------------------------------------------------------- + * Function: test_skip_compress_write2 + * + * Purpose: Test skipping compression filter when there are three filters + * for the dataset + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +static int +test_skip_compress_write2(hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int i, j, n; + + unsigned filter_mask = 0; /* orig filter mask */ + int origin_direct_buf[CHUNK_NX][CHUNK_NY]; + int direct_buf[CHUNK_NX][CHUNK_NY]; + int check_chunk[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + int aggression = 9; /* Compression aggression setting */ + + unsigned read_filter_mask = 0; /* filter mask after direct read */ + int read_direct_buf[CHUNK_NX][CHUNK_NY]; + hsize_t read_buf_size = 0; /* buf size */ + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("skipping compression filters but keep two other filters"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression. + * The order of filters is bogus 1 + deflate + bogus 2. + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + /* Register and enable first bogus filter */ + if(H5Zregister (H5Z_BOGUS1) < 0) + goto error; + + if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS1, 0, (size_t)0, NULL) < 0) + goto error; + + /* Enable compression filter */ + if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) + goto error; + + /* Register and enable second bogus filter */ + if(H5Zregister (H5Z_BOGUS2) < 0) + goto error; + + if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS2, 0, (size_t)0, NULL) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME3, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Initialize data for one chunk. Apply operations of two bogus filters to the chunk */ + for(i = n = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) { + origin_direct_buf[i][j] = n++; + direct_buf[i][j] = (origin_direct_buf[i][j] + ADD_ON) * FACTOR; + } + + /* write the uncompressed chunk data repeatedly to dataset, using the direct writing function. + * Indicate skipping the compression filter but keep the other two bogus filters */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + /* compression filter is the middle one to be skipped */ + filter_mask = 0x00000002; + + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) + goto error; + + if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + if((dataset = H5Dopen2(file, DATASETNAME3, H5P_DEFAULT)) < 0) + goto error; + + /* + * Select hyperslab for one chunk in the file + */ + start[0] = CHUNK_NX; start[1] = CHUNK_NY; + stride[0] = 1; stride[1] = 1; + count[0] = 1; count[1] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; + if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) + goto error; + + /* Read the chunk back */ + if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) + goto error; + + /* Check that the values read are the same as the values written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if(origin_direct_buf[i][j] != check_chunk[i][j]) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" origin_direct_buf=%d, check_chunk=%d\n", origin_direct_buf[i][j], check_chunk[i][j]); + goto error; + } + } + } + + /* Query chunk storage size */ + if((status = H5Dget_chunk_storage_size(dataset, offset, &read_buf_size)) < 0) + goto error; + if(read_buf_size != buf_size) + goto error; + + /* Read the raw chunk back */ + HDmemset(&read_direct_buf, 0, sizeof(read_direct_buf)); + if((status = H5Dread_chunk(dataset, H5P_DEFAULT, offset, &read_filter_mask, read_direct_buf)) < 0) + goto error; + if(read_filter_mask != filter_mask) + goto error; + + /* Check that the direct chunk read is the same as the chunk written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if(direct_buf[i][j] != read_direct_buf[i][j]) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" direct_buf=%d, read_direct_buf=%d\n", direct_buf[i][j], read_direct_buf[i][j]); + goto error; + } + } + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + } H5E_END_TRY; + + H5_FAILED(); + return 1; +} /* test_skip_compress_write2() */ + +/*------------------------------------------------------------------------- + * Function: test_data_conv + * + * Purpose: Test data conversion + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +static int +test_data_conv(hid_t file) +{ + typedef struct { + int a, b, c[4], d, e; + } src_type_t; + typedef struct { + int a, c[4], e; + } dst_type_t; + + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int i, j, n; + const hsize_t four = 4; + hid_t st=-1, dt=-1; + hid_t array_dt; + + unsigned filter_mask = 0; + src_type_t direct_buf[CHUNK_NX][CHUNK_NY]; + dst_type_t check_chunk[CHUNK_NX][CHUNK_NY]; + src_type_t read_chunk[CHUNK_NX][CHUNK_NY]; /* For H5Dread_chunk */ + + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(src_type_t); + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("data conversion for H5Dwrite_chunk/H5Dread_chunk"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties, i.e. enable chunking + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + /* Build hdf5 datatypes */ + array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); + if((st = H5Tcreate(H5T_COMPOUND, sizeof(src_type_t))) < 0 || + H5Tinsert(st, "a", HOFFSET(src_type_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(st, "b", HOFFSET(src_type_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(st, "c", HOFFSET(src_type_t, c), array_dt) < 0 || + H5Tinsert(st, "d", HOFFSET(src_type_t, d), H5T_NATIVE_INT) < 0 || + H5Tinsert(st, "e", HOFFSET(src_type_t, e), H5T_NATIVE_INT) < 0) + goto error; + + if(H5Tclose(array_dt) < 0) + goto error; + + array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); + if((dt = H5Tcreate(H5T_COMPOUND, sizeof(dst_type_t))) < 0 || + H5Tinsert(dt, "a", HOFFSET(dst_type_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(dt, "c", HOFFSET(dst_type_t, c), array_dt) < 0 || + H5Tinsert(dt, "e", HOFFSET(dst_type_t, e), H5T_NATIVE_INT) < 0) + goto error; + + if(H5Tclose(array_dt) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME4, st, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Initialize data for one chunk */ + for(i = n = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + (direct_buf[i][j]).a = i*j+0; + (direct_buf[i][j]).b = i*j+1; + (direct_buf[i][j]).c[0] = i*j+2; + (direct_buf[i][j]).c[1] = i*j+3; + (direct_buf[i][j]).c[2] = i*j+4; + (direct_buf[i][j]).c[3] = i*j+5; + (direct_buf[i][j]).d = i*j+6; + (direct_buf[i][j]).e = i*j+7; + } + } + + /* write the chunk data to dataset, using the direct writing function. + * There should be no data conversion involved. */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) + goto error; + + if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + if((dataset = H5Dopen2(file, DATASETNAME4, H5P_DEFAULT)) < 0) + goto error; + + /* Use H5Dread_chunk() to read the uncompressed data */ + if((status = H5Dread_chunk(dataset, dxpl, offset, &filter_mask, read_chunk)) < 0) + goto error; + + /* Check that the values read are the same as the values written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if ((direct_buf[i][j]).a != (read_chunk[i][j]).a || + (direct_buf[i][j]).b != (read_chunk[i][j]).b || + (direct_buf[i][j]).c[0] != (read_chunk[i][j]).c[0] || + (direct_buf[i][j]).c[1] != (read_chunk[i][j]).c[1] || + (direct_buf[i][j]).c[2] != (read_chunk[i][j]).c[2] || + (direct_buf[i][j]).c[3] != (read_chunk[i][j]).c[3] || + (direct_buf[i][j]).d != (read_chunk[i][j]).d || + (direct_buf[i][j]).e != (read_chunk[i][j]).e) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], + (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); + HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + (read_chunk[i][j]).a, (read_chunk[i][j]).b, (read_chunk[i][j]).c[0], (read_chunk[i][j]).c[1], + (read_chunk[i][j]).c[2], (read_chunk[i][j]).c[3], (read_chunk[i][j]).d, (read_chunk[i][j]).e); + + goto error; + } + } + } + + /* + * Select hyperslab for the chunk just written in the file + */ + start[0] = CHUNK_NX; start[1] = CHUNK_NY; + stride[0] = 1; stride[1] = 1; + count[0] = 1; count[1] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; + if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) + goto error; + + /* Read the chunk back. Data should be converted */ + if((status = H5Dread(dataset, dt, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) + goto error; + + /* Check that the values read are the same as the values written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if ((direct_buf[i][j]).a != (check_chunk[i][j]).a || + (direct_buf[i][j]).c[0] != (check_chunk[i][j]).c[0] || + (direct_buf[i][j]).c[1] != (check_chunk[i][j]).c[1] || + (direct_buf[i][j]).c[2] != (check_chunk[i][j]).c[2] || + (direct_buf[i][j]).c[3] != (check_chunk[i][j]).c[3] || + (direct_buf[i][j]).e != (check_chunk[i][j]).e) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], + (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); + HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n", + (check_chunk[i][j]).a, (check_chunk[i][j]).c[0], (check_chunk[i][j]).c[1], (check_chunk[i][j]).c[2], + (check_chunk[i][j]).c[3], (check_chunk[i][j]).e); + + goto error; + } + } + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + H5Tclose(st); + H5Tclose(dt); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + H5Tclose(st); + H5Tclose(dt); + } H5E_END_TRY; + + H5_FAILED(); + return 1; +} /* test_data_conv() */ + +/*------------------------------------------------------------------------- + * Function: test_invalid_parameters + * + * Purpose: Test invalid parameters for H5Dwrite_chunk and H5Dread_chunk + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +static int +test_invalid_parameters(hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int i, j, n; + + unsigned filter_mask = 0; + int direct_buf[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + int aggression = 9; /* Compression aggression setting */ + + hsize_t chunk_nbytes; /* Chunk size */ + + TESTING("invalid parameters for H5Dwrite_chunk/H5Dread_chunk"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + /* + * Create a new contiguous dataset to verify H5Dwrite_chunk/H5Dread_chunk doesn't work + */ + if((dataset = H5Dcreate2(file, DATASETNAME5, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Initialize data for one chunk */ + for(i = n = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) { + direct_buf[i][j] = n++; + } + + /* Try to write the chunk data to contiguous dataset. It should fail */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Try to get chunk size for a contiguous dataset. It should fail */ + H5E_BEGIN_TRY { + if((status = H5Dget_chunk_storage_size(dataset, offset, &chunk_nbytes)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Try to H5Dread_chunk from the contiguous dataset. It should fail */ + H5E_BEGIN_TRY { + if((status = H5Dread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + if(H5Dclose(dataset) < 0) + goto error; + + + /* Create a chunked dataset with compression filter */ + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + if((status = H5Pset_deflate( cparms, (unsigned ) aggression)) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME6, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + /* Check invalid dataset ID for H5Dwrite_chunk and H5Dread_chunk */ + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk((hid_t)-1, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5Dread_chunk((hid_t)-1, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid DXPL ID for H5Dwrite_chunk and H5Dread_chunk */ + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk(dataset, (hid_t)-1, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5Dread_chunk(dataset, (hid_t)-1, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid OFFSET for H5Dwrite_chunk and H5Dread_chunk */ + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, NULL, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5Dread_chunk(dataset, dxpl, NULL, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check when OFFSET is out of dataset range for H5Dwrite_chunk and H5Dread_chunk */ + offset[0] = NX + 1; + offset[1] = NY; + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5Dread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check when OFFSET is not on chunk boundary for H5Dwrite_chunk and H5Dread_chunk */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY + 1; + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5Dread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid buffer size for H5Dwrite_chunk only */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + buf_size = 0; + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid data buffer for H5Dwrite_chunk and H5Dread_chunk */ + buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + H5E_BEGIN_TRY { + if((status = H5Dwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, NULL)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5Dread_chunk(dataset, dxpl, offset, &filter_mask, NULL)) != FAIL) + goto error; + } H5E_END_TRY; + + if(H5Dclose(dataset) < 0) + goto error; + + /* + * Close/release resources. + */ + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + } H5E_END_TRY; + + H5_FAILED(); + return 1; +} /* test_invalid_parameters() */ + +/*------------------------------------------------------------------------- + * Function: test_direct_chunk_read_no_cache + * + * Purpose: Test the basic functionality of H5Dread_chunk with the + * chunk cache diabled. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Matthew Strong (GE Healthcare) + * 14 February 2016 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_FILTER_DEFLATE +static int +test_direct_chunk_read_no_cache (hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1, dapl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; + herr_t status; /* status from H5 function calls */ + int ret; /* deflate return status */ + int data[NX][NY]; + int i, j, k, l, n; /* local index variables */ + + unsigned filter_mask = 0; /* filter mask returned from H5Dread_chunk */ + int direct_buf[CHUNK_NX][CHUNK_NY]; /* chunk read with H5Dread and manually decompressed */ + int check_chunk[CHUNK_NX][CHUNK_NY]; /* chunk read with H5Dread */ + hsize_t offset[2]; /* chunk offset used for H5Dread_chunk */ + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + + Bytef *z_src = NULL; /* source buffer */ + uLongf z_src_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + Bytef *z_dst = (Bytef*)(direct_buf); + uLong z_dst_nbytes = (uLong)buf_size; + int aggression = 9; /* Compression aggression setting */ + void *outbuf = NULL; /* Pointer to new buffer */ + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("basic functionality of H5Dread_chunk (chunk cache disabled)"); + + /* Create the data space with unlimited dimensions. */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* Modify dataset creation properties, i.e. enable chunking and compression */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) + goto error; + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + + /* Disable chunk cache by setting number of slots to 0 */ + if((status = H5Pset_chunk_cache(dapl, 0, H5D_CHUNK_CACHE_NBYTES_DEFAULT, H5D_CHUNK_CACHE_W0_DEFAULT)) < 0) + goto error; + + /* Create a new dataset within the file using cparms creation properties. */ + if((dataset = H5Dcreate2(file, DATASETNAME8, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, dapl)) < 0) + goto error; + + /* Initialize the dataset */ + for(i = n = 0; i < NX; i++) + for(j = 0; j < NY; j++) + data[i][j] = n++; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Write the data for the dataset. + * Data will skip chunk cache and go directly to disk. */ + if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + dxpl, data)) < 0) + goto error; + + /* Allocate output (compressed) buffer */ + outbuf = HDmalloc(z_src_nbytes); + z_src = (Bytef *)outbuf; + + /* For each chunk in the dataset, compare the result of H5Dread and H5Dread_chunk. */ + for(i=0; i + +#if !defined(WIN32) && !defined(__MINGW32__) + +#include + +#ifdef H5_STDC_HEADERS +# include +# include +# include +# include +#endif + +#ifdef H5_HAVE_UNISTD_H +# include +# include +#endif + +#ifdef H5_HAVE_SYS_STAT_H +# include +#endif + +#if defined(H5_TIME_WITH_SYS_TIME) +# include +# include +#elif defined(H5_HAVE_SYS_TIME_H) +# include +#else +# include +#endif + +const char *FILENAME[] = { + "direct_write", + "unix.raw", + NULL +}; + +/* + * Print the current location on the standard output stream. + */ +#define FUNC __func__ +#define AT() printf (" at %s:%d in %s()...\n", \ + __FILE__, __LINE__, FUNC); +#define H5_FAILED() {puts("*FAILED*");fflush(stdout);} +#define TEST_ERROR {H5_FAILED(); AT(); goto error;} +#define TESTING(WHAT) {printf("Testing %-62s",WHAT); fflush(stdout);} +#define PASSED() {puts(" PASSED");fflush(stdout);} + +#define DIRECT_UNCOMPRESSED_DSET "direct_uncompressed_dset" +#define DIRECT_COMPRESSED_DSET "direct_compressed_dset" +#define REG_COMPRESSED_DSET "reg_compressed_dset" +#define REG_NO_COMPRESS_DSET "reg_no_compress_dset" +#define RANK 3 +#define NX 100 +#define NY 1000 +#define NZ 250 +#define CHUNK_NX 1 +#define CHUNK_NY 1000 +#define CHUNK_NZ 250 + +#define DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s))*1.001)+12) +char filename[1024]; +unsigned int *outbuf[NX]; +size_t data_size[NX]; +double total_size = 0.0; +unsigned int *direct_buf[NX]; +double MB = 1048576.0; + +/*-------------------------------------------------- + * Function to report IO rate + *-------------------------------------------------- + */ +void reportTime(struct timeval start, double mbytes) +{ + struct timeval timeval_stop,timeval_diff; + + /*end timing*/ + gettimeofday(&timeval_stop,NULL); + + /* Calculate the elapsed gettimeofday time */ + timeval_diff.tv_usec=timeval_stop.tv_usec-start.tv_usec; + timeval_diff.tv_sec=timeval_stop.tv_sec-start.tv_sec; + + if(timeval_diff.tv_usec<0) { + timeval_diff.tv_usec+=1000000; + timeval_diff.tv_sec--; + } /* end if */ + +/*printf("mbytes=%lf, sec=%lf, usec=%lf\n", mbytes, (double)timeval_diff.tv_sec, (double)timeval_diff.tv_usec);*/ + printf("MBytes/second: %lf\n", (double)mbytes/((double)timeval_diff.tv_sec+((double)timeval_diff.tv_usec/(double)1000000.0))); +} + +/*-------------------------------------------------- + * Create file, datasets, and initialize data + *-------------------------------------------------- + */ +int create_file(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t fapl; + hid_t cparms; + hid_t dataspace, dataset; + hsize_t dims[RANK] = {NX, NY, NZ}; + hsize_t chunk_dims[RANK] ={CHUNK_NX, CHUNK_NY, CHUNK_NZ}; + unsigned int aggression = 9; /* Compression aggression setting */ + int ret; + int i, j, n; + + int flag; + int unix_file; + + unsigned int *p; + size_t buf_size = CHUNK_NY*CHUNK_NZ*sizeof(unsigned int); + + const Bytef *z_src; + Bytef *z_dst; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + uLong z_src_nbytes = (uLong)buf_size; + + TESTING("Create a file and dataset"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* + * Create a new file. If file exists its contents will be overwritten. + */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + TEST_ERROR; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + if(H5Pset_chunk( cparms, RANK, chunk_dims) < 0) + TEST_ERROR; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DIRECT_UNCOMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + if((dataset = H5Dcreate2(file, REG_NO_COMPRESS_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + /* Set compression */ + if(H5Pset_deflate( cparms, aggression) < 0) + TEST_ERROR; + + if((dataset = H5Dcreate2(file, DIRECT_COMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + + if((dataset = H5Dcreate2(file, REG_COMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + if(H5Fclose(file) < 0) + TEST_ERROR; + + if(H5Sclose(dataspace) < 0) + TEST_ERROR; + + if(H5Pclose(cparms) < 0) + TEST_ERROR; + + /* create a unix file*/ + flag = O_CREAT|O_TRUNC|O_WRONLY; + + if ((unix_file=open(FILENAME[1],flag,S_IRWXU))== -1) + TEST_ERROR; + + if (close(unix_file) < 0) + { + printf(" unable to close the file\n"); + TEST_ERROR; + } + + + /* Initialize data for chunks */ + for(i = 0; i < NX; i++) { + p = direct_buf[i] = (unsigned int*)malloc(CHUNK_NY*CHUNK_NZ*sizeof(unsigned int)); + + for(j=0; j < CHUNK_NY*CHUNK_NZ; j++, p++) + *p = rand() % 65000; + + z_src = (const Bytef*)direct_buf[i]; + + z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + /* Allocate output (compressed) buffer */ + outbuf[i] = (unsigned int*)malloc((size_t)z_dst_nbytes); + z_dst = (Bytef *)outbuf[i]; + + /* Perform compression from the source to the destination buffer */ + ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + data_size[i] = (size_t)z_dst_nbytes; + total_size += data_size[i]; + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == ret) { + fprintf(stderr, "overflow"); + TEST_ERROR; + } else if(Z_MEM_ERROR == ret) { + fprintf(stderr, "deflate memory error"); + TEST_ERROR; + } else if(Z_OK != ret) { + fprintf(stderr, "other deflate error"); + TEST_ERROR; + } + } + + + PASSED(); + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} + +/*-------------------------------------------------- + * Benchmark the performance of the new function + * with precompressed data. + *-------------------------------------------------- + */ +int +test_direct_write_uncompressed_data(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t dataspace, dataset; + hid_t dxpl; + herr_t status; + int i; + + unsigned filter_mask = 0; + hsize_t offset[RANK] = {0, 0, 0}; + + struct timeval timeval_start; + + TESTING("H5Dwrite_chunk for uncompressed data"); + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + /* Start the timer */ + gettimeofday(&timeval_start,NULL); + + /* Reopen the file and dataset */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + + if((dataset = H5Dopen2(file, DIRECT_UNCOMPRESSED_DSET, H5P_DEFAULT)) < 0) + TEST_ERROR; + + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i Date: Mon, 30 Apr 2018 09:34:53 -0700 Subject: Finished move of H5DOread/write_chunk calls to H5D. --- src/H5CX.c | 280 --------------------------------------- src/H5CXprivate.h | 7 - src/H5Dio.c | 388 ++++++++++++++++++++---------------------------------- src/H5Dpkg.h | 2 - src/H5Pdxpl.c | 65 --------- 5 files changed, 142 insertions(+), 600 deletions(-) diff --git a/src/H5CX.c b/src/H5CX.c index deaf1a1..f809d1c 100644 --- a/src/H5CX.c +++ b/src/H5CX.c @@ -233,24 +233,10 @@ typedef struct H5CX_t { hbool_t data_transform_valid; /* Whether data transform info is valid */ H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ hbool_t vl_alloc_info_valid; /* Whether VL datatype alloc info is valid */ - hbool_t dcr_flag; /* Direct chunk read flag (H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME) */ - hbool_t dcr_flag_valid; /* Whether direct chunk read flag is valid */ - hsize_t *dcr_offset; /* Direct chunk read offset (H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME) */ - hbool_t dcr_offset_valid; /* Whether direct chunk read offset is valid */ - hbool_t dcw_flag; /* Direct chunk write flag (H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME) */ - hbool_t dcw_flag_valid; /* Whether direct chunk write flag is valid */ - uint32_t dcw_filters; /* Direct chunk write filter flags (H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME) */ - hbool_t dcw_filters_valid; /* Whether direct chunk write filter flags is valid */ - hsize_t *dcw_offset; /* Direct chunk write offset (H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME) */ - hbool_t dcw_offset_valid; /* Whether direct chunk write offset is valid */ - uint32_t dcw_datasize; /* Direct chunk write data size (H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME) */ - hbool_t dcw_datasize_valid; /* Whether direct chunk write data size is valid */ H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ hbool_t dt_conv_cb_valid; /* Whether datatype conversion struct is valid */ /* Return-only DXPL properties to return to application */ - uint32_t dcr_filters; /* Direct chunk read filter flags (H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME) */ - hbool_t dcr_filters_set; /* Whether direct chunk read filter flags are set */ #ifdef H5_HAVE_PARALLEL H5D_mpio_actual_chunk_opt_mode_t mpio_actual_chunk_opt; /* Chunk optimization mode used for parallel I/O (H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME) */ hbool_t mpio_actual_chunk_opt_set; /* Whether chunk optimization mode used for parallel I/O is set */ @@ -321,12 +307,6 @@ typedef struct H5CX_dxpl_cache_t { H5Z_cb_t filter_cb; /* Filter callback function (H5D_XFER_FILTER_CB_NAME) */ H5Z_data_xform_t *data_transform; /* Data transform info (H5D_XFER_XFORM_NAME) */ H5T_vlen_alloc_info_t vl_alloc_info; /* VL datatype alloc info (H5D_XFER_VLEN_*_NAME) */ - hbool_t dcr_flag; /* Direct chunk read flag (H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME) */ - hsize_t *dcr_offset; /* Direct chunk read offset (H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME) */ - hbool_t dcw_flag; /* Direct chunk write flag (H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME) */ - uint32_t dcw_datasize; /* Direct chunk write data size (H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME) */ - hsize_t *dcw_offset; /* Direct chunk write offset (H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME) */ - uint32_t dcw_filters; /* Direct chunk write filter flags (H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME) */ H5T_conv_cb_t dt_conv_cb; /* Datatype conversion struct (H5D_XFER_CONV_CB_NAME) */ } H5CX_dxpl_cache_t; @@ -469,27 +449,10 @@ H5CX__init_package(void) if(H5P_get(dx_plist, H5D_XFER_VLEN_FREE_INFO_NAME, &H5CX_def_dxpl_cache.vl_alloc_info.free_info) < 0) HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve VL datatype alloc info") - /* Get direct chunk read info */ - if(H5P_get(dx_plist, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, &H5CX_def_dxpl_cache.dcr_flag) < 0) - HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve direct chunk read flag") - if(H5P_get(dx_plist, H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME, &H5CX_def_dxpl_cache.dcr_offset) < 0) - HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve direct chunk read offset") - - /* Get direct chunk write info */ - if(H5P_get(dx_plist, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &H5CX_def_dxpl_cache.dcw_flag) < 0) - HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve direct chunk write flag") - if(H5P_get(dx_plist, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, &H5CX_def_dxpl_cache.dcw_filters) < 0) - HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve direct chunk write filter mask") - if(H5P_get(dx_plist, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, &H5CX_def_dxpl_cache.dcw_offset) < 0) - HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve direct chunk write offset") - if(H5P_get(dx_plist, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, &H5CX_def_dxpl_cache.dcw_datasize) < 0) - HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve direct chunk write data size") - /* Get datatype conversion struct */ if(H5P_get(dx_plist, H5D_XFER_CONV_CB_NAME, &H5CX_def_dxpl_cache.dt_conv_cb) < 0) HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve datatype conversion exception callback") - /* Reset the "default LAPL cache" information */ HDmemset(&H5CX_def_lapl_cache, 0, sizeof(H5CX_lapl_cache_t)); @@ -1791,216 +1754,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5CX_get_dcr_flag - * - * Purpose: Retrieves the direct chunk read flag for the current API call context. - * - * Return: Non-negative on success / Negative on failure - * - * Programmer: Quincey Koziol - * March 6, 2018 - * - *------------------------------------------------------------------------- - */ -herr_t -H5CX_get_dcr_flag(hbool_t *dcr_flag) -{ - H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity check */ - HDassert(dcr_flag); - HDassert(head && *head); - HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); - - H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, dcr_flag) - - /* Get the value */ - *dcr_flag = (*head)->ctx.dcr_flag; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5CX_get_dcr_flag() */ - - -/*------------------------------------------------------------------------- - * Function: H5CX_get_dcr_offset - * - * Purpose: Retrieves the direct chunk read offset for the current API call context. - * - * Return: Non-negative on success / Negative on failure - * - * Programmer: Quincey Koziol - * March 6, 2018 - * - *------------------------------------------------------------------------- - */ -herr_t -H5CX_get_dcr_offset(hsize_t **dcr_offset) -{ - H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity check */ - HDassert(dcr_offset); - HDassert(head && *head); - HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); - - H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME, dcr_offset) - - /* Get the value */ - *dcr_offset = (*head)->ctx.dcr_offset; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5CX_get_dcr_offset() */ - - -/*------------------------------------------------------------------------- - * Function: H5CX_get_dcw_flag - * - * Purpose: Retrieves the direct chunk write flag for the current API call context. - * - * Return: Non-negative on success / Negative on failure - * - * Programmer: Quincey Koziol - * March 6, 2018 - * - *------------------------------------------------------------------------- - */ -herr_t -H5CX_get_dcw_flag(hbool_t *dcw_flag) -{ - H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity check */ - HDassert(dcw_flag); - HDassert(head && *head); - HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); - - H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, dcw_flag) - - /* Get the value */ - *dcw_flag = (*head)->ctx.dcw_flag; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5CX_get_dcw_flag() */ - - -/*------------------------------------------------------------------------- - * Function: H5CX_get_dcw_filters - * - * Purpose: Retrieves the direct chunk write filter mask for the current API call context. - * - * Return: Non-negative on success / Negative on failure - * - * Programmer: Quincey Koziol - * March 6, 2018 - * - *------------------------------------------------------------------------- - */ -herr_t -H5CX_get_dcw_filters(uint32_t *dcw_filters) -{ - H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity check */ - HDassert(dcw_filters); - HDassert(head && *head); - HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); - - H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, dcw_filters) - - /* Get the value */ - *dcw_filters = (*head)->ctx.dcw_filters; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5CX_get_dcw_filters() */ - - -/*------------------------------------------------------------------------- - * Function: H5CX_get_dcw_offset - * - * Purpose: Retrieves the direct chunk write offset for the current API call context. - * - * Return: Non-negative on success / Negative on failure - * - * Programmer: Quincey Koziol - * March 6, 2018 - * - *------------------------------------------------------------------------- - */ -herr_t -H5CX_get_dcw_offset(hsize_t **dcw_offset) -{ - H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity check */ - HDassert(dcw_offset); - HDassert(head && *head); - HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); - - H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, dcw_offset) - - /* Get the value */ - *dcw_offset = (*head)->ctx.dcw_offset; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5CX_get_dcw_offset() */ - - -/*------------------------------------------------------------------------- - * Function: H5CX_get_dcw_datasize - * - * Purpose: Retrieves the direct chunk write data size for the current API call context. - * - * Return: Non-negative on success / Negative on failure - * - * Programmer: Quincey Koziol - * March 6, 2018 - * - *------------------------------------------------------------------------- - */ -herr_t -H5CX_get_dcw_datasize(uint32_t *dcw_datasize) -{ - H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity check */ - HDassert(dcw_datasize); - HDassert(head && *head); - HDassert(H5P_DEFAULT != (*head)->ctx.dxpl_id); - - H5CX_RETRIEVE_PROP_VALID(dxpl, H5P_DATASET_XFER_DEFAULT, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, dcw_datasize) - - /* Get the value */ - *dcw_datasize = (*head)->ctx.dcw_datasize; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5CX_get_dcw_datasize() */ - - -/*------------------------------------------------------------------------- * Function: H5CX_get_dt_conv_cb * * Purpose: Retrieves the datatype conversion exception callback for the current API call context. @@ -2358,38 +2111,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_nlinks() */ - -/*------------------------------------------------------------------------- - * Function: H5CX_set_dcr_filters - * - * Purpose: Sets the direct chunk read filter flags for the current API call context. - * - * Return: - * - * Programmer: Quincey Koziol - * March 6, 2018 - * - *------------------------------------------------------------------------- - */ -void -H5CX_set_dcr_filters(uint32_t direct_filters) -{ - H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Sanity checks */ - HDassert(head && *head); - HDassert(!((*head)->ctx.dxpl_id == H5P_DEFAULT || - (*head)->ctx.dxpl_id == H5P_DATASET_XFER_DEFAULT)); - - /* Cache the filter mask for later, marking it to set in DXPL when context popped */ - (*head)->ctx.dcr_filters = direct_filters; - (*head)->ctx.dcr_filters_set = TRUE; - - FUNC_LEAVE_NOAPI_VOID -} /* end H5CX_set_dcr_filters() */ - #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- @@ -2755,7 +2476,6 @@ H5CX__pop_common(void) HDassert(head && *head); /* Check for cached DXPL properties to return to application */ - H5CX_SET_PROP(H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME, dcr_filters) #ifdef H5_HAVE_PARALLEL H5CX_SET_PROP(H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, mpio_actual_chunk_opt) H5CX_SET_PROP(H5D_MPIO_ACTUAL_IO_MODE_NAME, mpio_actual_io_mode) diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index 3566f11..44a4067 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -95,12 +95,6 @@ H5_DLL herr_t H5CX_get_err_detect(H5Z_EDC_t *err_detect); H5_DLL herr_t H5CX_get_filter_cb(H5Z_cb_t *filter_cb); H5_DLL herr_t H5CX_get_data_transform(H5Z_data_xform_t **data_transform); H5_DLL herr_t H5CX_get_vlen_alloc_info(H5T_vlen_alloc_info_t *vl_alloc_info); -H5_DLL herr_t H5CX_get_dcr_flag(hbool_t *direct_read); -H5_DLL herr_t H5CX_get_dcr_offset(hsize_t **direct_offset); -H5_DLL herr_t H5CX_get_dcw_flag(hbool_t *direct_write); -H5_DLL herr_t H5CX_get_dcw_filters(uint32_t *direct_filters); -H5_DLL herr_t H5CX_get_dcw_offset(hsize_t **direct_offset); -H5_DLL herr_t H5CX_get_dcw_datasize(uint32_t *direct_datasize); H5_DLL herr_t H5CX_get_dt_conv_cb(H5T_conv_cb_t *cb_struct); /* "Getter" routines for LAPL properties cached in API context */ @@ -127,7 +121,6 @@ H5_DLL herr_t H5CX_set_vlen_alloc_info(H5MM_allocate_t alloc_func, H5_DLL herr_t H5CX_set_nlinks(size_t nlinks); /* "Setter" routines for cached DXPL properties that must be returned to application */ -H5_DLL void H5CX_set_dcr_filters(uint32_t direct_filters); #ifdef H5_HAVE_PARALLEL H5_DLL void H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chunk_opt); H5_DLL void H5CX_set_mpio_actual_io_mode(H5D_mpio_actual_io_mode_t actual_io_mode); diff --git a/src/H5Dio.c b/src/H5Dio.c index 70a0bfe..6dec5b0 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -113,102 +113,56 @@ herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t dxpl_id, void *buf/*out*/) { - H5D_t *dset = NULL; - const H5S_t *mem_space = NULL; - const H5S_t *file_space = NULL; - hbool_t direct_read = FALSE; - herr_t ret_value = SUCCEED; /* Return value */ + H5D_t *dset = NULL; + const H5S_t *mem_space = NULL; + const H5S_t *file_space = NULL; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE6("e", "iiiiix", dset_id, mem_type_id, mem_space_id, file_space_id, dxpl_id, buf); /* check arguments */ - if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + if (NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dset_id is not a dataset ID") - if(NULL == dset->oloc.file) + if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file") - if(mem_space_id < 0) + if (mem_space_id < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid mem_space_id") - if(file_space_id < 0) + if (file_space_id < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file_space_id") - if(H5S_ALL != mem_space_id) { - if(NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) + /* Check dataspaces */ + if (H5S_ALL != mem_space_id) { + if (NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space") /* Check for valid selection */ - if(H5S_SELECT_VALID(mem_space) != TRUE) + if (H5S_SELECT_VALID(mem_space) != TRUE) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent") - } /* end if */ - - if(H5S_ALL != file_space_id) { - if(NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) + } + if (H5S_ALL != file_space_id) { + if (NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space") /* Check for valid selection */ - if(H5S_SELECT_VALID(file_space) != TRUE) + if (H5S_SELECT_VALID(file_space) != TRUE) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection+offset not within extent") - } /* end if */ + } /* Get the default dataset transfer property list if the user didn't provide one */ - if(H5P_DEFAULT == dxpl_id) + if (H5P_DEFAULT == dxpl_id) dxpl_id = H5P_DATASET_XFER_DEFAULT; else - if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") /* Set DXPL for operation */ H5CX_set_dxpl(dxpl_id); - /* Retrieve the 'direct read' flag */ - if(H5CX_get_dcr_flag(&direct_read) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting flag for direct chunk read") - - /* Set up for direct read of chunk, bypassing filters, etc. */ - if(direct_read) { - hsize_t *direct_offset; /* Chunk offset from calling routine */ - hsize_t internal_offset[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ - uint32_t direct_filters = 0; /* Filters for chunk */ - unsigned u; /* Local index variable */ - - /* Sanity check */ - if(H5D_CHUNKED != dset->shared->layout.type) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") - - /* Get the direct chunk offset */ - if(H5CX_get_dcr_offset(&direct_offset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting offset for direct chunk read") - HDassert(direct_offset); - - /* The library's chunking code requires the offset terminates with a zero. So transfer the - * offset array to an internal offset array */ - for(u = 0; u < dset->shared->ndims; u++) { - /* Make sure the offset doesn't exceed the dataset's dimensions */ - if(direct_offset[u] > dset->shared->curr_dims[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") - - /* Make sure the offset fall right on a chunk's boundary */ - if(direct_offset[u] % dset->shared->layout.u.chunk.dim[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") - - internal_offset[u] = direct_offset[u]; - } /* end for */ - - /* Terminate the offset with a zero */ - internal_offset[dset->shared->ndims] = 0; - - /* Read the raw chunk */ - if(H5D__chunk_direct_read(dset, internal_offset, &direct_filters, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read chunk directly") - - /* Set the chunk filter mask for application */ - H5CX_set_dcr_filters(direct_filters); - } /* end if */ - else - /* Read raw data */ - if(H5D__read(dset, mem_type_id, mem_space, file_space, buf/*out*/) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data") + /* Read raw data */ + if (H5D__read(dset, mem_type_id, mem_space, file_space, buf/*out*/) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data") done: FUNC_LEAVE_API(ret_value) @@ -231,56 +185,62 @@ herr_t H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *filters, void *buf) { - hbool_t created_dxpl = FALSE; /* Whether we created a DXPL */ - hbool_t do_direct_read = TRUE; /* Flag for direct writes */ - herr_t ret_value = SUCCEED; /* Return value */ + H5D_t *dset = NULL; + hsize_t internal_offset[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE5("e", "ii*h*Iu*x", dset_id, dxpl_id, offset, filters, buf); /* Check arguments */ - if(dset_id < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset ID") - if(!buf) + if (NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dset_id is not a dataset ID") + if (NULL == dset->oloc.file) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file") + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + if (!buf) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf cannot be NULL") - if(!offset) + if (!offset) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL") - if(!filters) + if (!filters) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filters cannot be NULL") - /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ - if(H5P_DEFAULT == dxpl_id) { - if((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, FAIL, "cannot create dxpl") - created_dxpl = TRUE; - } /* end if */ + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID") - /* Set direct write parameters */ - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, &do_direct_read) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set direct read property") - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME, &offset) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set offset property") + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); - /* Read chunk */ - if(H5Dread(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "cannot read from dataset") + /* The library's chunking code requires the offset to terminate with a zero. + * So transfer the offset array to an internal offset array that we + * can properly terminate (don't mess with the passed-in buffer). + */ + for (u = 0; u < dset->shared->ndims; u++) { + /* Make sure the offset doesn't exceed the dataset's dimensions */ + if (offset[u] > dset->shared->curr_dims[u]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") - /* Get the filter mask */ - if(H5Pget(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME, filters) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "cannot get filter mask property") + /* Make sure the offset fall right on a chunk's boundary */ + if (offset[u] % dset->shared->layout.u.chunk.dim[u]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") -done: - if(created_dxpl) { - if(H5Pclose(dxpl_id) < 0) - HDONE_ERROR(H5E_PLIST, H5E_CANTRELEASE, FAIL, "unable to close dxpl") - } /* end if */ - else { - /* Reset the direct read flag on user DXPL */ - do_direct_read = FALSE; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, &do_direct_read) < 0) - HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot reset direct write property") + internal_offset[u] = offset[u]; } + /* Terminate the offset with a zero */ + internal_offset[dset->shared->ndims] = 0; + + /* Read the raw chunk */ + if (H5D__chunk_direct_read(dset, internal_offset, filters, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data") + +done: FUNC_LEAVE_API(ret_value) } /* end H5Dread_chunk() */ @@ -320,10 +280,9 @@ herr_t H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t dxpl_id, const void *buf) { - H5D_t *dset = NULL; + H5D_t *dset = NULL; const H5S_t *mem_space = NULL; const H5S_t *file_space = NULL; - hbool_t direct_write = FALSE; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -331,50 +290,46 @@ H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, dxpl_id, buf); /* check arguments */ - if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") - if(NULL == dset->oloc.file) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + if (NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dset_id is not a dataset ID") + if (NULL == dset->oloc.file) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file") + if (mem_space_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid mem_space_id") + if (file_space_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file_space_id") + + /* Check dataspace selections */ + if (H5S_ALL != mem_space_id) { + if (NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "mem_space_id is not a dataspace ID") + + /* Check for valid selection */ + if (H5S_SELECT_VALID(mem_space) != TRUE) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "memory selection + offset not within extent") + } + if (H5S_ALL != file_space_id) { + if (NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "file_space_id is not a dataspace ID") + + /* Check for valid selection */ + if (H5S_SELECT_VALID(file_space) != TRUE) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "file selection + offset not within extent") + } /* Get the default dataset transfer property list if the user didn't provide one */ - if(H5P_DEFAULT == dxpl_id) + if (H5P_DEFAULT == dxpl_id) dxpl_id = H5P_DATASET_XFER_DEFAULT; else - if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms") /* Set DXPL for operation */ H5CX_set_dxpl(dxpl_id); - /* Retrieve the 'direct write' flag */ - if(H5CX_get_dcw_flag(&direct_write) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting flag for direct chunk read") - - /* Check dataspace selections if this is not a direct write */ - if(!direct_write) { - if(mem_space_id < 0 || file_space_id < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") - - if(H5S_ALL != mem_space_id) { - if(NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") - - /* Check for valid selection */ - if(H5S_SELECT_VALID(mem_space) != TRUE) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "memory selection+offset not within extent") - } /* end if */ - if(H5S_ALL != file_space_id) { - if(NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") - - /* Check for valid selection */ - if(H5S_SELECT_VALID(file_space) != TRUE) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "file selection+offset not within extent") - } /* end if */ - } /* end if */ - - if(H5D__pre_write(dset, direct_write, mem_type_id, mem_space, file_space, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't prepare for writing data") + /* Write the data */ + if (H5D__write(dset, mem_type_id, mem_space, file_space, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data") done: FUNC_LEAVE_API(ret_value) @@ -397,129 +352,70 @@ herr_t H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, size_t data_size, const void *buf) { - hbool_t created_dxpl = FALSE; /* Whether we created a DXPL */ - hbool_t do_direct_write = TRUE; /* Flag for direct writes */ - uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ - herr_t ret_value = SUCCEED; /* Return value */ + H5D_t *dset = NULL; + hsize_t internal_offset[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + unsigned u; /* Local index variable */ + uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE6("e", "iiIu*hz*x", dset_id, dxpl_id, filters, offset, data_size, buf); /* Check arguments */ - if(dset_id < 0) + if (NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset ID") - if(!buf) + if (NULL == dset->oloc.file) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file") + if (H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + if (!buf) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "buf cannot be NULL") - if(!offset) + if (!offset) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "offset cannot be NULL") - if(!data_size) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "data_size cannot be NULL") + if (0 == data_size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "data_size cannot be zero") + + /* Make sure data size is less than 4 GiB */ data_size_32 = (uint32_t)data_size; - if(data_size != (size_t)data_size_32) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid data_size") - - /* If the user passed in a default DXPL, create one to pass to H5Dwrite() */ - if(H5P_DEFAULT == dxpl_id) { - if((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, FAIL, "cannot create dxpl") - created_dxpl = TRUE; - } /* end if */ + if (data_size != (size_t)data_size_32) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid data_size - chunks cannot be > 4 GiB") - /* Set direct write parameters */ - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set direct read property") - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, &filters) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set filters property") - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, &offset) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set offset property") - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, &data_size_32) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot set data size property") + /* Get the default dataset transfer property list if the user didn't provide one */ + if (H5P_DEFAULT == dxpl_id) + dxpl_id = H5P_DATASET_XFER_DEFAULT; + else + if (TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dxpl_id is not a dataset transfer property list ID") - /* Write chunk */ - if(H5Dwrite(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "cannot write to dataset") + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); -done: - if(created_dxpl) { - if(H5Pclose(dxpl_id) < 0) - HDONE_ERROR(H5E_PLIST, H5E_CANTRELEASE, FAIL, "unable to close dxpl") - } /* end if */ - else { - /* Reset the direct write flag on user DXPL */ - do_direct_write = FALSE; - if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) - HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "cannot reset direct write property") - } + /* The library's chunking code requires the offset to terminate with a zero. + * So transfer the offset array to an internal offset array that we + * can properly terminate (don't mess with the passed-in buffer). + */ + for (u = 0; u < dset->shared->ndims; u++) { + /* Make sure the offset doesn't exceed the dataset's dimensions */ + if (offset[u] > dset->shared->curr_dims[u]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") - FUNC_LEAVE_API(ret_value) -} /* end H5Dwrite_chunk() */ + /* Make sure the offset fall right on a chunk's boundary */ + if (offset[u] % dset->shared->layout.u.chunk.dim[u]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") - -/*------------------------------------------------------------------------- - * Function: H5D__pre_write - * - * Purpose: Preparation for writing data. - * - * Return: SUCCEED/FAIL - * - *------------------------------------------------------------------------- - */ -herr_t -H5D__pre_write(H5D_t *dset, hbool_t direct_write, hid_t mem_type_id, - const H5S_t *mem_space, const H5S_t *file_space, const void *buf) -{ - herr_t ret_value = SUCCEED; /* Return value */ + internal_offset[u] = offset[u]; + } - FUNC_ENTER_PACKAGE_VOL - - /* Direct chunk write */ - if(direct_write) { - uint32_t direct_filters; - hsize_t *direct_offset; - uint32_t direct_datasize; - hsize_t internal_offset[H5O_LAYOUT_NDIMS]; - unsigned u; /* Local index variable */ - - if(H5D_CHUNKED != dset->shared->layout.type) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") - - /* Retrieve parameters for direct chunk write */ - if(H5CX_get_dcw_filters(&direct_filters) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting filter info for direct chunk write") - if(H5CX_get_dcw_offset(&direct_offset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting offset info for direct chunk write") - if(H5CX_get_dcw_datasize(&direct_datasize) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting data size for direct chunk write") - - /* The library's chunking code requires the offset terminates with a zero. So transfer the - * offset array to an internal offset array */ - for(u = 0; u < dset->shared->ndims; u++) { - /* Make sure the offset doesn't exceed the dataset's dimensions */ - if(direct_offset[u] > dset->shared->curr_dims[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") - - /* Make sure the offset fall right on a chunk's boundary */ - if(direct_offset[u] % dset->shared->layout.u.chunk.dim[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") - - internal_offset[u] = direct_offset[u]; - } /* end for */ - - /* Terminate the offset with a zero */ - internal_offset[dset->shared->ndims] = 0; - - /* write raw data */ - if(H5D__chunk_direct_write(dset, direct_filters, internal_offset, direct_datasize, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write chunk directly") - } /* end if */ - else - /* Normal write of raw data */ - if(H5D__write(dset, mem_type_id, mem_space, file_space, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data") + /* Terminate the offset with a zero */ + internal_offset[dset->shared->ndims] = 0; + + /* Write chunk */ + if (H5D__chunk_direct_write(dset, filters, internal_offset, data_size, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data") done: - FUNC_LEAVE_NOAPI_VOL(ret_value) -} /* end H5D__pre_write() */ + FUNC_LEAVE_API(ret_value) +} /* end H5Dwrite_chunk() */ /*------------------------------------------------------------------------- diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 126a3c0..b887b87 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -583,8 +583,6 @@ H5_DLL herr_t H5D__refresh(hid_t dset_id, H5D_t *dataset); H5_DLL herr_t H5D__format_convert(H5D_t *dataset); /* Internal I/O routines */ -H5_DLL herr_t H5D__pre_write(H5D_t *dset, hbool_t direct_write, hid_t mem_type_id, - const H5S_t *mem_space, const H5S_t *file_space, const void *buf); H5_DLL herr_t H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, void *buf/*out*/); H5_DLL herr_t H5D__write(H5D_t *dataset, hid_t mem_type_id, diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index e33e762..b56c137 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -152,22 +152,6 @@ #define H5D_XFER_XFORM_COPY H5P__dxfr_xform_copy #define H5D_XFER_XFORM_CMP H5P__dxfr_xform_cmp #define H5D_XFER_XFORM_CLOSE H5P__dxfr_xform_close -/* Definitions for properties of direct chunk write */ -#define H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_SIZE sizeof(hbool_t) -#define H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_DEF FALSE -#define H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_SIZE sizeof(uint32_t) -#define H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_DEF 0 -#define H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_SIZE sizeof(hsize_t *) -#define H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_DEF NULL -#define H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_SIZE sizeof(uint32_t) -#define H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_DEF 0 -/* Definitions for properties of direct chunk read */ -#define H5D_XFER_DIRECT_CHUNK_READ_FLAG_SIZE sizeof(hbool_t) -#define H5D_XFER_DIRECT_CHUNK_READ_FLAG_DEF FALSE -#define H5D_XFER_DIRECT_CHUNK_READ_FILTERS_SIZE sizeof(uint32_t) -#define H5D_XFER_DIRECT_CHUNK_READ_FILTERS_DEF 0 -#define H5D_XFER_DIRECT_CHUNK_READ_OFFSET_SIZE sizeof(hsize_t *) -#define H5D_XFER_DIRECT_CHUNK_READ_OFFSET_DEF NULL /******************/ @@ -266,13 +250,6 @@ static const H5Z_EDC_t H5D_def_enable_edc_g = H5D_XFER_EDC_DEF; /* De static const H5Z_cb_t H5D_def_filter_cb_g = H5D_XFER_FILTER_CB_DEF; /* Default value for filter callback */ static const H5T_conv_cb_t H5D_def_conv_cb_g = H5D_XFER_CONV_CB_DEF; /* Default value for datatype conversion callback */ static const void *H5D_def_xfer_xform_g = H5D_XFER_XFORM_DEF; /* Default value for data transform */ -static const hbool_t H5D_def_direct_chunk_flag_g = H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_DEF; /* Default value for the flag of direct chunk write */ -static const uint32_t H5D_def_direct_chunk_filters_g = H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_DEF; /* Default value for the filters of direct chunk write */ -static const hsize_t *H5D_def_direct_chunk_offset_g = H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_DEF; /* Default value for the offset of direct chunk write */ -static const uint32_t H5D_def_direct_chunk_datasize_g = H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_DEF; /* Default value for the datasize of direct chunk write */ -static const hbool_t direct_chunk_read_flag = H5D_XFER_DIRECT_CHUNK_READ_FLAG_DEF; /* Default value for the flag of direct chunk read */ -static const hsize_t *direct_chunk_read_offset = H5D_XFER_DIRECT_CHUNK_READ_OFFSET_DEF; /* Default value for the offset of direct chunk read */ -static const uint32_t direct_chunk_read_filters = H5D_XFER_DIRECT_CHUNK_READ_FILTERS_DEF; /* Default value for the filters of direct chunk read */ /*------------------------------------------------------------------------- @@ -423,48 +400,6 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) H5D_XFER_XFORM_DEL, H5D_XFER_XFORM_COPY, H5D_XFER_XFORM_CMP, H5D_XFER_XFORM_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - /* Register the property of flag for direct chunk write */ - /* (Note: this property should not have an encode/decode callback -QAK) */ - if(H5P_register_real(pclass, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_SIZE, &H5D_def_direct_chunk_flag_g, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - - /* Register the property of filter for direct chunk write */ - /* (Note: this property should not have an encode/decode callback -QAK) */ - if(H5P_register_real(pclass, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_SIZE, &H5D_def_direct_chunk_filters_g, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - - /* Register the property of offset for direct chunk write */ - /* (Note: this property should not have an encode/decode callback -QAK) */ - if(H5P_register_real(pclass, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_SIZE, &H5D_def_direct_chunk_offset_g, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - - /* Register the property of datasize for direct chunk write */ - /* (Note: this property should not have an encode/decode callback -QAK) */ - if(H5P_register_real(pclass, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_SIZE, &H5D_def_direct_chunk_datasize_g, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - - /* Register the property of flag for direct chunk read */ - /* (Note: this property should not have an encode/decode callback) */ - if(H5P_register_real(pclass, H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME, H5D_XFER_DIRECT_CHUNK_READ_FLAG_SIZE, &direct_chunk_read_flag, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - - /* Register the property of filter for direct chunk read */ - /* (Note: this property should not have an encode/decode callback) */ - if(H5P_register_real(pclass, H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME, H5D_XFER_DIRECT_CHUNK_READ_FILTERS_SIZE, &direct_chunk_read_filters, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - - /* Register the property of offset for direct chunk read */ - /* (Note: this property should not have an encode/decode callback) */ - if(H5P_register_real(pclass, H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME, H5D_XFER_DIRECT_CHUNK_READ_OFFSET_SIZE, &direct_chunk_read_offset, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dxfr_reg_prop() */ -- cgit v0.12 From 4fe4394c5856336345efd2e7293d703c9bfc6d1b Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 30 Apr 2018 15:04:45 -0700 Subject: Fixed a warning. --- src/H5Dio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5Dio.c b/src/H5Dio.c index 6dec5b0..b1117e9 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -410,7 +410,7 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of internal_offset[dset->shared->ndims] = 0; /* Write chunk */ - if (H5D__chunk_direct_write(dset, filters, internal_offset, data_size, buf) < 0) + if (H5D__chunk_direct_write(dset, filters, internal_offset, data_size_32, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data") done: -- cgit v0.12 From 3313fde3bb8b6cc5cfa3c8401615ac064f7cbdf9 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 30 Apr 2018 15:15:14 -0700 Subject: Normalize with trunk prior to update merge --- src/H5Dio.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/H5Dio.c b/src/H5Dio.c index b1117e9..1447f99 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -90,14 +90,14 @@ H5FL_DEFINE(H5D_chunk_map_t); * passed to this function with the PLIST_ID argument. * * The FILE_SPACE_ID can be the constant H5S_ALL which indicates - * that the entire file data space is to be referenced. + * that the entire file dataspace is to be referenced. * * The MEM_SPACE_ID can be the constant H5S_ALL in which case - * the memory data space is the same as the file data space + * the memory dataspace is the same as the file dataspace * defined when the dataset was created. * - * The number of elements in the memory data space must match - * the number of elements in the file data space. + * The number of elements in the memory dataspace must match + * the number of elements in the file dataspace. * * The PLIST_ID can be the constant H5P_DEFAULT in which * case the default data transfer properties are used. @@ -135,7 +135,7 @@ H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, /* Check dataspaces */ if (H5S_ALL != mem_space_id) { if (NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") /* Check for valid selection */ if (H5S_SELECT_VALID(mem_space) != TRUE) @@ -143,7 +143,7 @@ H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, } if (H5S_ALL != file_space_id) { if (NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") /* Check for valid selection */ if (H5S_SELECT_VALID(file_space) != TRUE) @@ -257,14 +257,14 @@ done: * PLIST_ID argument. * * The FILE_SPACE_ID can be the constant H5S_ALL which indicates - * that the entire file data space is to be referenced. + * that the entire file dataspace is to be referenced. * * The MEM_SPACE_ID can be the constant H5S_ALL in which case - * the memory data space is the same as the file data space + * the memory dataspace is the same as the file dataspace * defined when the dataset was created. * - * The number of elements in the memory data space must match - * the number of elements in the file data space. + * The number of elements in the memory dataspace must match + * the number of elements in the file dataspace. * * The PLIST_ID can be the constant H5P_DEFAULT in which * case the default data transfer properties are used. @@ -441,7 +441,7 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, hbool_t type_info_init = FALSE; /* Whether the datatype info has been initialized */ H5S_t * projected_mem_space = NULL; /* If not NULL, ptr to dataspace containing a */ /* projection of the supplied mem_space to a new */ - /* data space with rank equal to that of */ + /* dataspace with rank equal to that of */ /* file_space. */ /* */ /* This field is only used if */ @@ -494,7 +494,7 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, /* Make certain that the number of elements in each selection is the same */ if(nelmts != (hsize_t)H5S_GET_SELECT_NPOINTS(file_space)) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest dataspaces have different sizes") /* Check for a NULL buffer, after the H5S_ALL dataspace selection has been handled */ if(NULL == buf) { @@ -653,7 +653,7 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, hbool_t type_info_init = FALSE; /* Whether the datatype info has been initialized */ H5S_t * projected_mem_space = NULL; /* If not NULL, ptr to dataspace containing a */ /* projection of the supplied mem_space to a new */ - /* data space with rank equal to that of */ + /* dataspace with rank equal to that of */ /* file_space. */ /* */ /* This field is only used if */ @@ -737,7 +737,7 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, /* Make certain that the number of elements in each selection is the same */ if(nelmts != (hsize_t)H5S_GET_SELECT_NPOINTS(file_space)) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest dataspaces have different sizes") /* Check for a NULL buffer, after the H5S_ALL dataspace selection has been handled */ if(NULL == buf) { @@ -797,7 +797,7 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, if(H5D__ioinfo_init(dataset, &type_info, &store, &io_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up I/O operation") - /* Allocate data space and initialize it if it hasn't been. */ + /* Allocate dataspace and initialize it if it hasn't been. */ if(nelmts > 0 && dataset->shared->dcpl_cache.efl.nused == 0 && !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage)) { hssize_t file_nelmts; /* Number of elements in file dataset's dataspace */ @@ -992,7 +992,7 @@ H5D__typeinfo_init(const H5D_t *dset, hid_t mem_type_id, hbool_t do_write, } /* end else */ /* - * Locate the type conversion function and data space conversion + * Locate the type conversion function and dataspace conversion * functions, and set up the element numbering information. If a data * type conversion is necessary then register datatype atoms. Data type * conversion is necessary if the user has set the `need_bkg' to a high -- cgit v0.12 From ac035bf3679dc539c6b35534f310539210322779 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Wed, 2 May 2018 10:11:49 -0700 Subject: * Fixed the error tests * Moved common functionality into helper functions --- src/H5Dio.c | 171 +++++++++++++++++++++----------------------- src/H5S.c | 53 +++++++++++++- src/H5Sprivate.h | 1 + test/testfiles/err_compat_1 | 2 +- test/testfiles/error_test_1 | 4 +- 5 files changed, 139 insertions(+), 92 deletions(-) diff --git a/src/H5Dio.c b/src/H5Dio.c index 95efa08..781bd17 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -50,7 +50,8 @@ /* Local Prototypes */ /********************/ -/* Setup/teardown routines */ +static herr_t H5D__get_offset_copy(const H5D_t *dset, const hsize_t *offset, + hsize_t **offset_copy/*out*/); static herr_t H5D__ioinfo_init(H5D_t *dset, const H5D_type_info_t *type_info, H5D_storage_t *store, H5D_io_info_t *io_info); static herr_t H5D__typeinfo_init(const H5D_t *dset, hid_t mem_type_id, @@ -80,6 +81,56 @@ H5FL_DEFINE(H5D_chunk_map_t); /*------------------------------------------------------------------------- + * Function: H5D__get_offset_copy + * + * Purpose: Gets a copy of the user's offset array that is guaraneteed + * to be suitable for use by the library. + * + * The caller must free the constructed offset array. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t **offset_copy) +{ + unsigned u; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(dset); + HDassert(offset); + HDassert(offset_copy); + + if (NULL == (*offset_copy = (hsize_t *)H5MM_calloc(H5O_LAYOUT_NDIMS * sizeof(hsize_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate offset_copy array") + + /* The library's chunking code requires the offset to terminate with a zero. + * So transfer the offset array to an internal offset array that we + * can properly terminate (handled via the calloc call). + */ + for (u = 0; u < dset->shared->ndims; u++) { + /* Make sure the offset doesn't exceed the dataset's dimensions */ + if (offset[u] > dset->shared->curr_dims[u]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") + + /* Make sure the offset fall right on a chunk's boundary */ + if (offset[u] % dset->shared->layout.u.chunk.dim[u]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") + + (*offset_copy)[u] = offset[u]; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5D__get_offset_copy() */ + + + +/*------------------------------------------------------------------------- * Function: H5Dread * * Purpose: Reads (part of) a DSET from the file into application @@ -122,33 +173,17 @@ H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, H5TRACE6("e", "iiiiix", dset_id, mem_type_id, mem_space_id, file_space_id, dxpl_id, buf); - /* check arguments */ + /* Get dataset pointer */ if (NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dset_id is not a dataset ID") if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file") - if (mem_space_id < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid mem_space_id") - if (file_space_id < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file_space_id") - - /* Check dataspaces */ - if (H5S_ALL != mem_space_id) { - if (NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "mem_space_id is not a dataspace ID") - - /* Check for valid selection */ - if (H5S_SELECT_VALID(mem_space) != TRUE) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection + offset not within extent") - } - if (H5S_ALL != file_space_id) { - if (NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "file_space_id is not a dataspace ID") - /* Check for valid selection */ - if (H5S_SELECT_VALID(file_space) != TRUE) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection + offset not within extent") - } + /* Get validated dataspace pointers */ + if (H5S_get_validated_dataspace(mem_space_id, &mem_space) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not get a validated dataspace from mem_space_id") + if (H5S_get_validated_dataspace(file_space_id, &file_space) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not get a validated dataspace from file_space_id") /* Get the default dataset transfer property list if the user didn't provide one */ if (H5P_DEFAULT == dxpl_id) @@ -186,9 +221,8 @@ H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *fil void *buf) { H5D_t *dset = NULL; - hsize_t internal_offset[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + hsize_t *offset_copy = NULL; /* Internal copy of chunk offset */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE5("e", "ii*h*Iu*x", dset_id, dxpl_id, offset, filters, buf); @@ -217,30 +251,19 @@ H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *fil /* Set DXPL for operation */ H5CX_set_dxpl(dxpl_id); - /* The library's chunking code requires the offset to terminate with a zero. - * So transfer the offset array to an internal offset array that we - * can properly terminate (don't mess with the passed-in buffer). + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). */ - for (u = 0; u < dset->shared->ndims; u++) { - /* Make sure the offset doesn't exceed the dataset's dimensions */ - if (offset[u] > dset->shared->curr_dims[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") - - /* Make sure the offset fall right on a chunk's boundary */ - if (offset[u] % dset->shared->layout.u.chunk.dim[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") - - internal_offset[u] = offset[u]; - } - - /* Terminate the offset with a zero */ - internal_offset[dset->shared->ndims] = 0; + if (H5D__get_offset_copy(dset, offset, &offset_copy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failure to copy offset array") /* Read the raw chunk */ - if (H5D__chunk_direct_read(dset, internal_offset, filters, buf) < 0) + if (H5D__chunk_direct_read(dset, offset_copy, filters, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data") done: + offset_copy = (hsize_t *)H5MM_xfree(offset_copy); + FUNC_LEAVE_API(ret_value) } /* end H5Dread_chunk() */ @@ -289,33 +312,17 @@ H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, H5TRACE6("e", "iiiii*x", dset_id, mem_type_id, mem_space_id, file_space_id, dxpl_id, buf); - /* check arguments */ + /* Get dataset pointer and ensure it's associated with a file */ if (NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dset_id is not a dataset ID") if (NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not associated with a file") - if (mem_space_id < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid mem_space_id") - if (file_space_id < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file_space_id") - - /* Check dataspace selections */ - if (H5S_ALL != mem_space_id) { - if (NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "mem_space_id is not a dataspace ID") - - /* Check for valid selection */ - if (H5S_SELECT_VALID(mem_space) != TRUE) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "memory selection + offset not within extent") - } - if (H5S_ALL != file_space_id) { - if (NULL == (file_space = (const H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "file_space_id is not a dataspace ID") - /* Check for valid selection */ - if (H5S_SELECT_VALID(file_space) != TRUE) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "file selection + offset not within extent") - } + /* Get validated dataspace pointers */ + if (H5S_get_validated_dataspace(mem_space_id, &mem_space) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not get a validated dataspace from mem_space_id") + if (H5S_get_validated_dataspace(file_space_id, &file_space) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not get a validated dataspace from file_space_id") /* Get the default dataset transfer property list if the user didn't provide one */ if (H5P_DEFAULT == dxpl_id) @@ -353,10 +360,9 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of size_t data_size, const void *buf) { H5D_t *dset = NULL; - hsize_t internal_offset[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ - unsigned u; /* Local index variable */ - uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ - herr_t ret_value = SUCCEED; /* Return value */ + hsize_t *offset_copy = NULL; /* Internal copy of chunk offset */ + uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE6("e", "iiIu*hz*x", dset_id, dxpl_id, filters, offset, data_size, buf); @@ -390,30 +396,19 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of /* Set DXPL for operation */ H5CX_set_dxpl(dxpl_id); - /* The library's chunking code requires the offset to terminate with a zero. - * So transfer the offset array to an internal offset array that we - * can properly terminate (don't mess with the passed-in buffer). + /* Copy the user's offset array so we can be sure it's terminated properly. + * (we don't want to mess with the user's buffer). */ - for (u = 0; u < dset->shared->ndims; u++) { - /* Make sure the offset doesn't exceed the dataset's dimensions */ - if (offset[u] > dset->shared->curr_dims[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") - - /* Make sure the offset fall right on a chunk's boundary */ - if (offset[u] % dset->shared->layout.u.chunk.dim[u]) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") - - internal_offset[u] = offset[u]; - } - - /* Terminate the offset with a zero */ - internal_offset[dset->shared->ndims] = 0; + if (H5D__get_offset_copy(dset, offset, &offset_copy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failure to copy offset array") /* Write chunk */ - if (H5D__chunk_direct_write(dset, filters, internal_offset, data_size_32, buf) < 0) + if (H5D__chunk_direct_write(dset, filters, offset_copy, data_size_32, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data") done: + offset_copy = (hsize_t *)H5MM_xfree(offset_copy); + FUNC_LEAVE_API(ret_value) } /* end H5Dwrite_chunk() */ diff --git a/src/H5S.c b/src/H5S.c index 3d09fa0..88c2f72 100644 --- a/src/H5S.c +++ b/src/H5S.c @@ -26,7 +26,6 @@ #include "H5Fprivate.h" /* Files */ #include "H5FLprivate.h" /* Free lists */ #include "H5Iprivate.h" /* IDs */ -#include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" /* Object headers */ #include "H5Spkg.h" /* Dataspaces */ @@ -208,6 +207,58 @@ H5S_term_package(void) FUNC_LEAVE_NOAPI(n) } /* end H5S_term_package() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_get_validiated_dataspace + PURPOSE + Get a pointer to a validated H5S_t pointer + USAGE + H5S_t *H5S_get_validated_space(dataspace_id, space) + hid_t space_id; IN: The ID of the dataspace + const H5S_t * space; OUT: A pointer to the dataspace + RETURNS + SUCCEED/FAIL + DESCRIPTION + Gets a pointer to a dataspace struct after validating it. The pointer + can be NULL (if the ID is H5S_ALL, for example). + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S_get_validated_dataspace(hid_t space_id, const H5S_t **space) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(space); + + if (space_id < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid space_id (ID cannot be a negative number)") + + if (H5S_ALL == space_id) { + /* No special dataspace struct for H5S_ALL */ + *space = NULL; + } + else { + /* Get the dataspace pointer */ + if (NULL == (*space = (const H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "space_id is not a dataspace ID") + + /* Check for valid selection */ + if (H5S_SELECT_VALID(*space) != TRUE) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection + offset not within extent") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5S_get_validated_dataspace() */ + /*-------------------------------------------------------------------------- NAME diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index b4ae1ff..15ce75d 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -215,6 +215,7 @@ H5_DLL herr_t H5S_set_extent_real(H5S_t *space, const hsize_t *size); H5_DLL herr_t H5S_set_extent_simple(H5S_t *space, unsigned rank, const hsize_t *dims, const hsize_t *max); H5_DLL H5S_t *H5S_create(H5S_class_t type); +H5_DLL herr_t H5S_get_validated_dataspace(hid_t space_id, const H5S_t **space/*out*/); H5_DLL H5S_t *H5S_create_simple(unsigned rank, const hsize_t dims[/*rank*/], const hsize_t maxdims[/*rank*/]); H5_DLL herr_t H5S_set_version(H5F_t *f, H5S_t *ds); diff --git a/test/testfiles/err_compat_1 b/test/testfiles/err_compat_1 index d471e13..8190665 100644 --- a/test/testfiles/err_compat_1 +++ b/test/testfiles/err_compat_1 @@ -54,6 +54,6 @@ HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): #001: (file name) line (number) in test_error2(): H5Dwrite shouldn't succeed major: Error API minor: Write failed - #002: (file name) line (number) in H5Dwrite(): not a dataset + #002: (file name) line (number) in H5Dwrite(): dset_id is not a dataset ID major: Invalid arguments to routine minor: Inappropriate type diff --git a/test/testfiles/error_test_1 b/test/testfiles/error_test_1 index b0b5085..02cfed8 100644 --- a/test/testfiles/error_test_1 +++ b/test/testfiles/error_test_1 @@ -21,7 +21,7 @@ Error Test-DIAG: Error detected in Error Program (1.0) thread (IDs): Testing error API based on data I/O HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): - #000: (file name) line (number) in H5Dwrite(): not a dataset + #000: (file name) line (number) in H5Dwrite(): dset_id is not a dataset ID major: Invalid arguments to routine minor: Inappropriate type Error Test-DIAG: Error detected in Error Program (1.0) thread (IDs): @@ -32,7 +32,7 @@ Error Test-DIAG: Error detected in Error Program (1.0) thread (IDs): major: Error in IO minor: Error in H5Dwrite HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): - #002: (file name) line (number) in H5Dwrite(): not a dataset + #002: (file name) line (number) in H5Dwrite(): dset_id is not a dataset ID major: Invalid arguments to routine minor: Inappropriate type -- cgit v0.12 From 1fa0c5a4b57e51788d95a73b965fdbd5a6d2af5c Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Fri, 4 May 2018 09:01:54 -0700 Subject: * Added H5DO compatibility functions. * Changed the offset copy to use an array on the stack. * Yanked some unused #defines. --- hl/src/H5DO.c | 63 +- hl/src/H5DOpublic.h | 6 + hl/test/CMakeLists.txt | 1 + hl/test/CMakeTests.cmake | 2 + hl/test/Makefile.am | 5 +- hl/test/test_h5do_compat.c | 2174 ++++++++++++++++++++++++++++++++++++++++++++ src/H5Dio.c | 107 ++- src/H5Dpublic.h | 11 - 8 files changed, 2292 insertions(+), 77 deletions(-) create mode 100644 hl/test/test_h5do_compat.c diff --git a/hl/src/H5DO.c b/hl/src/H5DO.c index 20d5a4a..3caa32b 100644 --- a/hl/src/H5DO.c +++ b/hl/src/H5DO.c @@ -11,11 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include -#include -#include -#include - /* High-level library internal header file */ #include "H5HLprivate2.h" @@ -24,6 +19,57 @@ /*------------------------------------------------------------------------- + * Function: H5DOwrite_chunk + * + * Purpose: Writes an entire chunk to the file directly. + * + * The H5DOwrite_chunk() call was moved to H5Dwrite_chunk. This + * simple wrapper remains so that people can still link to the + * high-level library without changing their code. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, + size_t data_size, const void *buf) +{ + /* Call underlying H5D function */ + if (H5Dwrite_chunk(dset_id, dxpl_id, filters, offset, data_size, buf) < 0) + return FAIL; + else + return SUCCEED; + +} /* end H5DOwrite_chunk() */ + + +/*------------------------------------------------------------------------- + * Function: H5DOread_chunk + * + * Purpose: Reads an entire chunk from the file directly. + * + * The H5DOread_chunk() call was moved to H5Dread_chunk. This + * simple wrapper remains so that people can still link to the + * high-level library without changing their code. + * + * Return: Non-negative on success/Negative on failure + * + *--------------------------------------------------------------------------- + */ +herr_t +H5DOread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *filters, + void *buf) +{ + /* Call underlying H5D function */ + if (H5Dread_chunk(dset_id, dxpl_id, offset, filters, buf) < 0) + return FAIL; + else + return SUCCEED; + } /* end H5DOread_chunk() */ + + +/*------------------------------------------------------------------------- * Function: H5DOappend() * * Purpose: To append elements to a dataset. @@ -113,7 +159,8 @@ H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, goto done; /* Adjust the dimension size of the requested dimension, - but first record the old dimension size */ + * but first record the old dimension size + */ old_size = size[axis]; size[axis] += extension; if(size[axis] < old_size) @@ -125,7 +172,7 @@ H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, /* Get the new dataspace of the dataset */ if(FAIL == (new_space_id = H5Dget_space(dset_id))) - goto done; + goto done; /* Select a hyperslab corresponding to the append operation */ for(u = 0 ; u < ndims ; u++) { @@ -181,7 +228,7 @@ H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, goto done; /* Do a dataset flush */ - if(H5Dflush(dset_id) < 0) + if(H5Dflush(dset_id) < 0) goto done; } /* end if */ } /* end if */ diff --git a/hl/src/H5DOpublic.h b/hl/src/H5DOpublic.h index 7f83a7f..0271938 100644 --- a/hl/src/H5DOpublic.h +++ b/hl/src/H5DOpublic.h @@ -28,6 +28,12 @@ extern "C" { H5_HLDLL herr_t H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, hid_t memtype, const void *buf); +/* Compatibility wrappers for functionality moved to H5D */ +H5_HLDLL herr_t H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, + const hsize_t *offset, size_t data_size, const void *buf); +H5_HLDLL herr_t H5DOread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, + uint32_t *filters /*out*/, void *buf /*out*/); + #ifdef __cplusplus } #endif diff --git a/hl/test/CMakeLists.txt b/hl/test/CMakeLists.txt index 58869da..4e0ae83 100644 --- a/hl/test/CMakeLists.txt +++ b/hl/test/CMakeLists.txt @@ -48,6 +48,7 @@ HL_ADD_EXE (test_table) HL_ADD_EXE (test_ds) HL_ADD_EXE (test_ld) HL_ADD_EXE (test_dset_append) +HL_ADD_EXE (test_h5do_compat) # test_packet has two source files add_executable (hl_test_packet test_packet.c test_packet_vlen.c) diff --git a/hl/test/CMakeTests.cmake b/hl/test/CMakeTests.cmake index 6f57a8c..4e945bc 100644 --- a/hl/test/CMakeTests.cmake +++ b/hl/test/CMakeTests.cmake @@ -79,6 +79,7 @@ add_test ( file_img1.h5 file_img2.h5 test_append.h5 + h5do_compat.h5 test_detach.h5 test_ds1.h5 test_ds2.h5 @@ -116,4 +117,5 @@ HL_add_test (test_ds) HL_add_test (test_packet) HL_add_test (test_ld) HL_add_test (test_dset_append) +HL_add_test (test_h5do_compat) diff --git a/hl/test/Makefile.am b/hl/test/Makefile.am index 35e680c..2e63438 100644 --- a/hl/test/Makefile.am +++ b/hl/test/Makefile.am @@ -27,7 +27,7 @@ LDADD=$(LIBH5_HL) $(LIBH5TEST) $(LIBHDF5) # Test programs. These are our main targets. They should be listed in the # order to be executed, generally most specific tests to least specific tests. TEST_PROG=test_lite test_image test_file_image test_table test_ds test_packet \ - test_ld test_dset_append + test_ld test_dset_append test_h5do_compat check_PROGRAMS=$(TEST_PROG) # These programs generate test files for the tests. They don't need to be @@ -45,7 +45,8 @@ endif CHECK_CLEANFILES+=combine_tables[1-2].h5 test_ds[1-9].h5 test_ds10.h5 \ test_image[1-3].h5 file_img[1-2].h5 test_lite[1-4].h5 test_table.h5 \ test_packet_table.h5 test_packet_compress.h5 test_detach.h5 \ - test_packet_table_vlen.h5 testfl_packet_table_vlen.h5 test_append.h5 + test_packet_table_vlen.h5 testfl_packet_table_vlen.h5 test_append.h5 \ + h5do_compat.h5 # Sources for test_packet executable test_packet_SOURCES=test_packet.c test_packet_vlen.c diff --git a/hl/test/test_h5do_compat.c b/hl/test/test_h5do_compat.c new file mode 100644 index 0000000..3ad4eaf --- /dev/null +++ b/hl/test/test_h5do_compat.c @@ -0,0 +1,2174 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Copyright by The HDF Group. * +* Copyright by the Board of Trustees of the University of Illinois. * +* All rights reserved. * +* * +* This file is part of HDF5. The full HDF5 copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "h5hltest.h" +#include "H5DOpublic.h" + +/* This test is a direct copy of the direct_chunk.c test from test/ but + * with the H5Dread/write_chunk calls replaced with H5DOread/write_chunk. + * This ensures that older code which uses the H5DO calls can still + * work with the newer library. + */ + +#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) +# define H5_ZLIB_HEADER "zlib.h" +#endif +#if defined(H5_ZLIB_HEADER) +# include H5_ZLIB_HEADER /* "zlib.h" */ +#endif + +#define FILE_NAME "h5do_compat.h5" + +/* Datasets for Direct Write tests */ +#define DATASETNAME1 "direct_write" +#define DATASETNAME2 "skip_one_filter" +#define DATASETNAME3 "skip_two_filters" +#define DATASETNAME4 "data_conv" +#define DATASETNAME5 "contiguous_dset" +#define DATASETNAME6 "invalid_argue" +#define DATASETNAME7 "overwrite_chunk" +/* Datasets for Direct Read tests */ +#define DATASETNAME8 "disabled_chunk_cache" +#define DATASETNAME9 "flush_chunk_cache" +#define DATASETNAME10 "read_w_valid_cache" +#define DATASETNAME11 "unallocated_chunk" +#define DATASETNAME12 "unfiltered_data" + +#define RANK 2 +#define NX 16 +#define NY 16 +#define CHUNK_NX 4 +#define CHUNK_NY 4 + +#define DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*H5_DOUBLE(1.001))+H5_DOUBLE(12.0)) + +/* Temporary filter IDs used for testing */ +#define H5Z_FILTER_BOGUS1 305 +#define H5Z_FILTER_BOGUS2 306 +#define ADD_ON 7 +#define FACTOR 3 + +/* Constants for the overwrite test */ +#define OVERWRITE_NDIMS 3 +#define OVERWRITE_CHUNK_NX 3 +#define OVERWRITE_CHUNK_2NX 6 +#define OVERWRITE_CHUNK_NY 2 +#define OVERWRITE_VALUE 42 + +/* Defines used in test_single_chunk_latest() */ +#define FILE "single_latest.h5" +#define DATASET "dataset" +#define DIM0 4 +#define DIM1 32 +#define CHUNK0 DIM0 +#define CHUNK1 DIM1 + +/* Local prototypes for filter functions */ +static size_t filter_bogus1(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_BOGUS1[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS1, /* Filter id number */ + 1, 1, /* Encoding and decoding enabled */ + "bogus1", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + filter_bogus1, /* The actual filter function */ +}}; + +const H5Z_class2_t H5Z_BOGUS2[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS2, /* Filter id number */ + 1, 1, /* Encoding and decoding enabled */ + "bogus2", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + filter_bogus2, /* The actual filter function */ +}}; + +/*------------------------------------------------------------------------- + * Function: test_direct_chunk_write + * + * Purpose: Test the basic functionality of H5DOwrite_chunk + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_FILTER_DEFLATE +static int +test_direct_chunk_write (hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int ret; + int data[NX][NY]; + int i, j, n; + + unsigned filter_mask = 0; + int direct_buf[CHUNK_NX][CHUNK_NY]; + int check_chunk[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + + const Bytef *z_src = (const Bytef*)(direct_buf); + Bytef *z_dst = NULL; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + uLong z_src_nbytes = (uLong)buf_size; + int aggression = 9; /* Compression aggression setting */ + void *outbuf = NULL; /* Pointer to new buffer */ + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("basic functionality of H5DOwrite_chunk"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME1, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + /* Initialize the dataset */ + for(i = n = 0; i < NX; i++) + for(j = 0; j < NY; j++) + data[i][j] = n++; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* + * Write the data for the dataset. It should stay in the chunk cache. + * It will be evicted from the cache by the H5DOwrite_chunk calls. + */ + if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + dxpl, data)) < 0) + goto error; + + /* Initialize data for one chunk */ + for(i = n = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) + direct_buf[i][j] = n++; + + /* Allocate output (compressed) buffer */ + outbuf = HDmalloc(z_dst_nbytes); + z_dst = (Bytef *)outbuf; + + /* Perform compression from the source to the destination buffer */ + ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == ret) { + HDfprintf(stderr, "overflow"); + goto error; + } else if(Z_MEM_ERROR == ret) { + HDfprintf(stderr, "deflate memory error"); + goto error; + } else if(Z_OK != ret) { + HDfprintf(stderr, "other deflate error"); + goto error; + } + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i0) { + *int_ptr++ -= (int)ADD_ON; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end if */ + else { /* write */ + /* Add the "add on" value to all the data values */ + while(buf_left>0) { + *int_ptr++ += (int)ADD_ON; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end else */ + + return nbytes; +} /* filter_bogus1() */ + +/*------------------------------------------------------------------------- + * Function: filter_bogus2 + * + * Purpose: A bogus filter that multiplies the original value by FACTOR. + * + * Return: Success: Data chunk size + * + * Programmer: Raymond Lu + * 30 November 2012 + *------------------------------------------------------------------------- + */ +static size_t +filter_bogus2(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, + const unsigned int H5_ATTR_UNUSED *cd_values, size_t nbytes, + size_t *buf_size, void **buf) +{ + int *int_ptr=(int *)*buf; /* Pointer to the data values */ + ssize_t buf_left=(ssize_t)*buf_size; /* Amount of data buffer left to process */ + + if(flags & H5Z_FLAG_REVERSE) { /* read */ + /* Substract the "add on" value to all the data values */ + while(buf_left>0) { + *int_ptr++ /= (int)FACTOR; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end if */ + else { /* write */ + /* Add the "add on" value to all the data values */ + while(buf_left>0) { + *int_ptr++ *= (int)FACTOR; + buf_left -= (ssize_t)sizeof(int); + } /* end while */ + } /* end else */ + + return nbytes; +} /* filter_bogus2() */ + +/*------------------------------------------------------------------------- + * Function: test_skip_compress_write2 + * + * Purpose: Test skipping compression filter when there are three filters + * for the dataset + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +static int +test_skip_compress_write2(hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int i, j, n; + + unsigned filter_mask = 0; /* orig filter mask */ + int origin_direct_buf[CHUNK_NX][CHUNK_NY]; + int direct_buf[CHUNK_NX][CHUNK_NY]; + int check_chunk[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + int aggression = 9; /* Compression aggression setting */ + + unsigned read_filter_mask = 0; /* filter mask after direct read */ + int read_direct_buf[CHUNK_NX][CHUNK_NY]; + hsize_t read_buf_size = 0; /* buf size */ + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("skipping compression filters but keep two other filters"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression. + * The order of filters is bogus 1 + deflate + bogus 2. + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + /* Register and enable first bogus filter */ + if(H5Zregister (H5Z_BOGUS1) < 0) + goto error; + + if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS1, 0, (size_t)0, NULL) < 0) + goto error; + + /* Enable compression filter */ + if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) + goto error; + + /* Register and enable second bogus filter */ + if(H5Zregister (H5Z_BOGUS2) < 0) + goto error; + + if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS2, 0, (size_t)0, NULL) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME3, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Initialize data for one chunk. Apply operations of two bogus filters to the chunk */ + for(i = n = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) { + origin_direct_buf[i][j] = n++; + direct_buf[i][j] = (origin_direct_buf[i][j] + ADD_ON) * FACTOR; + } + + /* write the uncompressed chunk data repeatedly to dataset, using the direct writing function. + * Indicate skipping the compression filter but keep the other two bogus filters */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + /* compression filter is the middle one to be skipped */ + filter_mask = 0x00000002; + + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) + goto error; + + if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + if((dataset = H5Dopen2(file, DATASETNAME3, H5P_DEFAULT)) < 0) + goto error; + + /* + * Select hyperslab for one chunk in the file + */ + start[0] = CHUNK_NX; start[1] = CHUNK_NY; + stride[0] = 1; stride[1] = 1; + count[0] = 1; count[1] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; + if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) + goto error; + + /* Read the chunk back */ + if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) + goto error; + + /* Check that the values read are the same as the values written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if(origin_direct_buf[i][j] != check_chunk[i][j]) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" origin_direct_buf=%d, check_chunk=%d\n", origin_direct_buf[i][j], check_chunk[i][j]); + goto error; + } + } + } + + /* Query chunk storage size */ + if((status = H5Dget_chunk_storage_size(dataset, offset, &read_buf_size)) < 0) + goto error; + if(read_buf_size != buf_size) + goto error; + + /* Read the raw chunk back */ + HDmemset(&read_direct_buf, 0, sizeof(read_direct_buf)); + if((status = H5DOread_chunk(dataset, H5P_DEFAULT, offset, &read_filter_mask, read_direct_buf)) < 0) + goto error; + if(read_filter_mask != filter_mask) + goto error; + + /* Check that the direct chunk read is the same as the chunk written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if(direct_buf[i][j] != read_direct_buf[i][j]) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" direct_buf=%d, read_direct_buf=%d\n", direct_buf[i][j], read_direct_buf[i][j]); + goto error; + } + } + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + } H5E_END_TRY; + + H5_FAILED(); + return 1; +} /* test_skip_compress_write2() */ + +/*------------------------------------------------------------------------- + * Function: test_data_conv + * + * Purpose: Test data conversion + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +static int +test_data_conv(hid_t file) +{ + typedef struct { + int a, b, c[4], d, e; + } src_type_t; + typedef struct { + int a, c[4], e; + } dst_type_t; + + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int i, j, n; + const hsize_t four = 4; + hid_t st=-1, dt=-1; + hid_t array_dt; + + unsigned filter_mask = 0; + src_type_t direct_buf[CHUNK_NX][CHUNK_NY]; + dst_type_t check_chunk[CHUNK_NX][CHUNK_NY]; + src_type_t read_chunk[CHUNK_NX][CHUNK_NY]; /* For H5DOread_chunk */ + + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(src_type_t); + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("data conversion for H5DOwrite_chunk/H5DOread_chunk"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties, i.e. enable chunking + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + /* Build hdf5 datatypes */ + array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); + if((st = H5Tcreate(H5T_COMPOUND, sizeof(src_type_t))) < 0 || + H5Tinsert(st, "a", HOFFSET(src_type_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(st, "b", HOFFSET(src_type_t, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(st, "c", HOFFSET(src_type_t, c), array_dt) < 0 || + H5Tinsert(st, "d", HOFFSET(src_type_t, d), H5T_NATIVE_INT) < 0 || + H5Tinsert(st, "e", HOFFSET(src_type_t, e), H5T_NATIVE_INT) < 0) + goto error; + + if(H5Tclose(array_dt) < 0) + goto error; + + array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); + if((dt = H5Tcreate(H5T_COMPOUND, sizeof(dst_type_t))) < 0 || + H5Tinsert(dt, "a", HOFFSET(dst_type_t, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(dt, "c", HOFFSET(dst_type_t, c), array_dt) < 0 || + H5Tinsert(dt, "e", HOFFSET(dst_type_t, e), H5T_NATIVE_INT) < 0) + goto error; + + if(H5Tclose(array_dt) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME4, st, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Initialize data for one chunk */ + for(i = n = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + (direct_buf[i][j]).a = i*j+0; + (direct_buf[i][j]).b = i*j+1; + (direct_buf[i][j]).c[0] = i*j+2; + (direct_buf[i][j]).c[1] = i*j+3; + (direct_buf[i][j]).c[2] = i*j+4; + (direct_buf[i][j]).c[3] = i*j+5; + (direct_buf[i][j]).d = i*j+6; + (direct_buf[i][j]).e = i*j+7; + } + } + + /* write the chunk data to dataset, using the direct writing function. + * There should be no data conversion involved. */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) + goto error; + + if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + if((dataset = H5Dopen2(file, DATASETNAME4, H5P_DEFAULT)) < 0) + goto error; + + /* Use H5DOread_chunk() to read the uncompressed data */ + if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, read_chunk)) < 0) + goto error; + + /* Check that the values read are the same as the values written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if ((direct_buf[i][j]).a != (read_chunk[i][j]).a || + (direct_buf[i][j]).b != (read_chunk[i][j]).b || + (direct_buf[i][j]).c[0] != (read_chunk[i][j]).c[0] || + (direct_buf[i][j]).c[1] != (read_chunk[i][j]).c[1] || + (direct_buf[i][j]).c[2] != (read_chunk[i][j]).c[2] || + (direct_buf[i][j]).c[3] != (read_chunk[i][j]).c[3] || + (direct_buf[i][j]).d != (read_chunk[i][j]).d || + (direct_buf[i][j]).e != (read_chunk[i][j]).e) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], + (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); + HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + (read_chunk[i][j]).a, (read_chunk[i][j]).b, (read_chunk[i][j]).c[0], (read_chunk[i][j]).c[1], + (read_chunk[i][j]).c[2], (read_chunk[i][j]).c[3], (read_chunk[i][j]).d, (read_chunk[i][j]).e); + + goto error; + } + } + } + + /* + * Select hyperslab for the chunk just written in the file + */ + start[0] = CHUNK_NX; start[1] = CHUNK_NY; + stride[0] = 1; stride[1] = 1; + count[0] = 1; count[1] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; + if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) + goto error; + + /* Read the chunk back. Data should be converted */ + if((status = H5Dread(dataset, dt, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) + goto error; + + /* Check that the values read are the same as the values written */ + for(i = 0; i < CHUNK_NX; i++) { + for(j = 0; j < CHUNK_NY; j++) { + if ((direct_buf[i][j]).a != (check_chunk[i][j]).a || + (direct_buf[i][j]).c[0] != (check_chunk[i][j]).c[0] || + (direct_buf[i][j]).c[1] != (check_chunk[i][j]).c[1] || + (direct_buf[i][j]).c[2] != (check_chunk[i][j]).c[2] || + (direct_buf[i][j]).c[3] != (check_chunk[i][j]).c[3] || + (direct_buf[i][j]).e != (check_chunk[i][j]).e) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", i, j); + HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], + (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); + HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n", + (check_chunk[i][j]).a, (check_chunk[i][j]).c[0], (check_chunk[i][j]).c[1], (check_chunk[i][j]).c[2], + (check_chunk[i][j]).c[3], (check_chunk[i][j]).e); + + goto error; + } + } + } + + /* + * Close/release resources. + */ + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + H5Tclose(st); + H5Tclose(dt); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + H5Tclose(st); + H5Tclose(dt); + } H5E_END_TRY; + + H5_FAILED(); + return 1; +} /* test_data_conv() */ + +/*------------------------------------------------------------------------- + * Function: test_invalid_parameters + * + * Purpose: Test invalid parameters for H5DOwrite_chunk and H5DOread_chunk + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Raymond Lu + * 30 November 2012 + * + *------------------------------------------------------------------------- + */ +static int +test_invalid_parameters(hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int i, j, n; + + unsigned filter_mask = 0; + int direct_buf[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + int aggression = 9; /* Compression aggression setting */ + + hsize_t chunk_nbytes; /* Chunk size */ + + TESTING("invalid parameters for H5DOwrite_chunk/H5DOread_chunk"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Modify dataset creation properties + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + /* + * Create a new contiguous dataset to verify H5DOwrite_chunk/H5DOread_chunk doesn't work + */ + if((dataset = H5Dcreate2(file, DATASETNAME5, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Initialize data for one chunk */ + for(i = n = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) { + direct_buf[i][j] = n++; + } + + /* Try to write the chunk data to contiguous dataset. It should fail */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Try to get chunk size for a contiguous dataset. It should fail */ + H5E_BEGIN_TRY { + if((status = H5Dget_chunk_storage_size(dataset, offset, &chunk_nbytes)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Try to H5DOread_chunk from the contiguous dataset. It should fail */ + H5E_BEGIN_TRY { + if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + if(H5Dclose(dataset) < 0) + goto error; + + + /* Create a chunked dataset with compression filter */ + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + if((status = H5Pset_deflate( cparms, (unsigned ) aggression)) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME6, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + /* Check invalid dataset ID for H5DOwrite_chunk and H5DOread_chunk */ + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk((hid_t)-1, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5DOread_chunk((hid_t)-1, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid DXPL ID for H5DOwrite_chunk and H5DOread_chunk */ + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk(dataset, (hid_t)-1, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5DOread_chunk(dataset, (hid_t)-1, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid OFFSET for H5DOwrite_chunk and H5DOread_chunk */ + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, NULL, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5DOread_chunk(dataset, dxpl, NULL, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check when OFFSET is out of dataset range for H5DOwrite_chunk and H5DOread_chunk */ + offset[0] = NX + 1; + offset[1] = NY; + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check when OFFSET is not on chunk boundary for H5DOwrite_chunk and H5DOread_chunk */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY + 1; + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid buffer size for H5DOwrite_chunk only */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + buf_size = 0; + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) + goto error; + } H5E_END_TRY; + + /* Check invalid data buffer for H5DOwrite_chunk and H5DOread_chunk */ + buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + H5E_BEGIN_TRY { + if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, NULL)) != FAIL) + goto error; + } H5E_END_TRY; + + H5E_BEGIN_TRY { + if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, NULL)) != FAIL) + goto error; + } H5E_END_TRY; + + if(H5Dclose(dataset) < 0) + goto error; + + /* + * Close/release resources. + */ + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(mem_space); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Pclose(dxpl); + } H5E_END_TRY; + + H5_FAILED(); + return 1; +} /* test_invalid_parameters() */ + +/*------------------------------------------------------------------------- + * Function: test_direct_chunk_read_no_cache + * + * Purpose: Test the basic functionality of H5DOread_chunk with the + * chunk cache diabled. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Matthew Strong (GE Healthcare) + * 14 February 2016 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_FILTER_DEFLATE +static int +test_direct_chunk_read_no_cache (hid_t file) +{ + hid_t dataspace = -1, dataset = -1; + hid_t mem_space = -1; + hid_t cparms = -1, dxpl = -1, dapl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; + herr_t status; /* status from H5 function calls */ + int ret; /* deflate return status */ + int data[NX][NY]; + int i, j, k, l, n; /* local index variables */ + + unsigned filter_mask = 0; /* filter mask returned from H5DOread_chunk */ + int direct_buf[CHUNK_NX][CHUNK_NY]; /* chunk read with H5Dread and manually decompressed */ + int check_chunk[CHUNK_NX][CHUNK_NY]; /* chunk read with H5Dread */ + hsize_t offset[2]; /* chunk offset used for H5DOread_chunk */ + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + + Bytef *z_src = NULL; /* source buffer */ + uLongf z_src_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + Bytef *z_dst = (Bytef*)(direct_buf); + uLong z_dst_nbytes = (uLong)buf_size; + int aggression = 9; /* Compression aggression setting */ + void *outbuf = NULL; /* Pointer to new buffer */ + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("basic functionality of H5DOread_chunk (chunk cache disabled)"); + + /* Create the data space with unlimited dimensions. */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* Modify dataset creation properties, i.e. enable chunking and compression */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) + goto error; + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + + /* Disable chunk cache by setting number of slots to 0 */ + if((status = H5Pset_chunk_cache(dapl, 0, H5D_CHUNK_CACHE_NBYTES_DEFAULT, H5D_CHUNK_CACHE_W0_DEFAULT)) < 0) + goto error; + + /* Create a new dataset within the file using cparms creation properties. */ + if((dataset = H5Dcreate2(file, DATASETNAME8, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, dapl)) < 0) + goto error; + + /* Initialize the dataset */ + for(i = n = 0; i < NX; i++) + for(j = 0; j < NY; j++) + data[i][j] = n++; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* Write the data for the dataset. + * Data will skip chunk cache and go directly to disk. */ + if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + dxpl, data)) < 0) + goto error; + + /* Allocate output (compressed) buffer */ + outbuf = HDmalloc(z_src_nbytes); + z_src = (Bytef *)outbuf; + + /* For each chunk in the dataset, compare the result of H5Dread and H5DOread_chunk. */ + for(i=0; ishared->ndims; u++) { /* Make sure the offset doesn't exceed the dataset's dimensions */ if (offset[u] > dset->shared->curr_dims[u]) @@ -120,7 +119,7 @@ H5D__get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t **offset_ if (offset[u] % dset->shared->layout.u.chunk.dim[u]) HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") - (*offset_copy)[u] = offset[u]; + offset_copy[u] = offset[u]; } done: @@ -133,30 +132,30 @@ done: /*------------------------------------------------------------------------- * Function: H5Dread * - * Purpose: Reads (part of) a DSET from the file into application - * memory BUF. The part of the dataset to read is defined with - * MEM_SPACE_ID and FILE_SPACE_ID. The data points are - * converted from their file type to the MEM_TYPE_ID specified. - * Additional miscellaneous data transfer properties can be - * passed to this function with the PLIST_ID argument. + * Purpose: Reads (part of) a DSET from the file into application + * memory BUF. The part of the dataset to read is defined with + * MEM_SPACE_ID and FILE_SPACE_ID. The data points are + * converted from their file type to the MEM_TYPE_ID specified. + * Additional miscellaneous data transfer properties can be + * passed to this function with the PLIST_ID argument. * - * The FILE_SPACE_ID can be the constant H5S_ALL which indicates - * that the entire file dataspace is to be referenced. + * The FILE_SPACE_ID can be the constant H5S_ALL which indicates + * that the entire file dataspace is to be referenced. * - * The MEM_SPACE_ID can be the constant H5S_ALL in which case - * the memory dataspace is the same as the file dataspace - * defined when the dataset was created. + * The MEM_SPACE_ID can be the constant H5S_ALL in which case + * the memory dataspace is the same as the file dataspace + * defined when the dataset was created. * - * The number of elements in the memory dataspace must match - * the number of elements in the file dataspace. + * The number of elements in the memory dataspace must match + * the number of elements in the file dataspace. * - * The PLIST_ID can be the constant H5P_DEFAULT in which - * case the default data transfer properties are used. + * The PLIST_ID can be the constant H5P_DEFAULT in which + * case the default data transfer properties are used. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * Thursday, December 4, 1997 + * Programmer: Robb Matzke + * Thursday, December 4, 1997 * *------------------------------------------------------------------------- */ @@ -221,8 +220,8 @@ H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *fil void *buf) { H5D_t *dset = NULL; - hsize_t *offset_copy = NULL; /* Internal copy of chunk offset */ - herr_t ret_value = SUCCEED; /* Return value */ + hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE5("e", "ii*h*Iu*x", dset_id, dxpl_id, offset, filters, buf); @@ -254,7 +253,7 @@ H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *fil /* Copy the user's offset array so we can be sure it's terminated properly. * (we don't want to mess with the user's buffer). */ - if (H5D__get_offset_copy(dset, offset, &offset_copy) < 0) + if (H5D__get_offset_copy(dset, offset, offset_copy) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failure to copy offset array") /* Read the raw chunk */ @@ -262,40 +261,38 @@ H5Dread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *fil HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data") done: - offset_copy = (hsize_t *)H5MM_xfree(offset_copy); - FUNC_LEAVE_API(ret_value) } /* end H5Dread_chunk() */ /*------------------------------------------------------------------------- - * Function: H5Dwrite + * Function: H5Dwrite * - * Purpose: Writes (part of) a DSET from application memory BUF to the - * file. The part of the dataset to write is defined with the - * MEM_SPACE_ID and FILE_SPACE_ID arguments. The data points - * are converted from their current type (MEM_TYPE_ID) to their - * file datatype. Additional miscellaneous data transfer - * properties can be passed to this function with the - * PLIST_ID argument. + * Purpose: Writes (part of) a DSET from application memory BUF to the + * file. The part of the dataset to write is defined with the + * MEM_SPACE_ID and FILE_SPACE_ID arguments. The data points + * are converted from their current type (MEM_TYPE_ID) to their + * file datatype. Additional miscellaneous data transfer + * properties can be passed to this function with the + * PLIST_ID argument. * - * The FILE_SPACE_ID can be the constant H5S_ALL which indicates - * that the entire file dataspace is to be referenced. + * The FILE_SPACE_ID can be the constant H5S_ALL which indicates + * that the entire file dataspace is to be referenced. * - * The MEM_SPACE_ID can be the constant H5S_ALL in which case - * the memory dataspace is the same as the file dataspace - * defined when the dataset was created. + * The MEM_SPACE_ID can be the constant H5S_ALL in which case + * the memory dataspace is the same as the file dataspace + * defined when the dataset was created. * - * The number of elements in the memory dataspace must match - * the number of elements in the file dataspace. + * The number of elements in the memory dataspace must match + * the number of elements in the file dataspace. * - * The PLIST_ID can be the constant H5P_DEFAULT in which - * case the default data transfer properties are used. + * The PLIST_ID can be the constant H5P_DEFAULT in which + * case the default data transfer properties are used. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * Thursday, December 4, 1997 + * Programmer: Robb Matzke + * Thursday, December 4, 1997 * *------------------------------------------------------------------------- */ @@ -360,9 +357,9 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of size_t data_size, const void *buf) { H5D_t *dset = NULL; - hsize_t *offset_copy = NULL; /* Internal copy of chunk offset */ - uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ - herr_t ret_value = SUCCEED; /* Return value */ + hsize_t offset_copy[H5O_LAYOUT_NDIMS]; /* Internal copy of chunk offset */ + uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE6("e", "iiIu*hz*x", dset_id, dxpl_id, filters, offset, data_size, buf); @@ -399,7 +396,7 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of /* Copy the user's offset array so we can be sure it's terminated properly. * (we don't want to mess with the user's buffer). */ - if (H5D__get_offset_copy(dset, offset, &offset_copy) < 0) + if (H5D__get_offset_copy(dset, offset, offset_copy) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failure to copy offset array") /* Write chunk */ @@ -407,8 +404,6 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write unprocessed chunk data") done: - offset_copy = (hsize_t *)H5MM_xfree(offset_copy); - FUNC_LEAVE_API(ret_value) } /* end H5Dwrite_chunk() */ diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index be80cef..6cec3f5 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -35,17 +35,6 @@ /* Bit flags for the H5Pset_chunk_opts() and H5Pget_chunk_opts() */ #define H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS (0x0002u) -/* Property names for H5DOwrite_chunk */ -#define H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME "direct_chunk_flag" -#define H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME "direct_chunk_filters" -#define H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME "direct_chunk_offset" -#define H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME "direct_chunk_datasize" - -/* Property names for H5DOread_chunk */ -#define H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME "direct_chunk_read_flag" -#define H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME "direct_chunk_read_offset" -#define H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME "direct_chunk_read_filters" - /*******************/ /* Public Typedefs */ /*******************/ -- cgit v0.12 From def636edb0ce9a2627015c336efe9f9f836c7101 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Fri, 4 May 2018 14:13:56 -0700 Subject: Stripped out most of the duplicated functionality in the H5DO compat test. --- hl/test/test_h5do_compat.c | 2201 +++----------------------------------------- 1 file changed, 151 insertions(+), 2050 deletions(-) diff --git a/hl/test/test_h5do_compat.c b/hl/test/test_h5do_compat.c index 3ad4eaf..486f9da 100644 --- a/hl/test/test_h5do_compat.c +++ b/hl/test/test_h5do_compat.c @@ -20,2155 +20,256 @@ * work with the newer library. */ -#if defined(H5_HAVE_ZLIB_H) && !defined(H5_ZLIB_HEADER) -# define H5_ZLIB_HEADER "zlib.h" -#endif -#if defined(H5_ZLIB_HEADER) -# include H5_ZLIB_HEADER /* "zlib.h" */ -#endif +#define FILE_NAME "h5do_compat.h5" +#define DATASET_NAME "direct_chunk_io" -#define FILE_NAME "h5do_compat.h5" - -/* Datasets for Direct Write tests */ -#define DATASETNAME1 "direct_write" -#define DATASETNAME2 "skip_one_filter" -#define DATASETNAME3 "skip_two_filters" -#define DATASETNAME4 "data_conv" -#define DATASETNAME5 "contiguous_dset" -#define DATASETNAME6 "invalid_argue" -#define DATASETNAME7 "overwrite_chunk" -/* Datasets for Direct Read tests */ -#define DATASETNAME8 "disabled_chunk_cache" -#define DATASETNAME9 "flush_chunk_cache" -#define DATASETNAME10 "read_w_valid_cache" -#define DATASETNAME11 "unallocated_chunk" -#define DATASETNAME12 "unfiltered_data" - -#define RANK 2 -#define NX 16 -#define NY 16 -#define CHUNK_NX 4 -#define CHUNK_NY 4 - -#define DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*H5_DOUBLE(1.001))+H5_DOUBLE(12.0)) - -/* Temporary filter IDs used for testing */ -#define H5Z_FILTER_BOGUS1 305 -#define H5Z_FILTER_BOGUS2 306 -#define ADD_ON 7 -#define FACTOR 3 - -/* Constants for the overwrite test */ -#define OVERWRITE_NDIMS 3 -#define OVERWRITE_CHUNK_NX 3 -#define OVERWRITE_CHUNK_2NX 6 -#define OVERWRITE_CHUNK_NY 2 -#define OVERWRITE_VALUE 42 - -/* Defines used in test_single_chunk_latest() */ -#define FILE "single_latest.h5" -#define DATASET "dataset" -#define DIM0 4 -#define DIM1 32 -#define CHUNK0 DIM0 -#define CHUNK1 DIM1 - -/* Local prototypes for filter functions */ -static size_t filter_bogus1(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); -static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); - -/* This message derives from H5Z */ -const H5Z_class2_t H5Z_BOGUS1[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - H5Z_FILTER_BOGUS1, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ - "bogus1", /* Filter name for debugging */ - NULL, /* The "can apply" callback */ - NULL, /* The "set local" callback */ - filter_bogus1, /* The actual filter function */ -}}; - -const H5Z_class2_t H5Z_BOGUS2[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - H5Z_FILTER_BOGUS2, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ - "bogus2", /* Filter name for debugging */ - NULL, /* The "can apply" callback */ - NULL, /* The "set local" callback */ - filter_bogus2, /* The actual filter function */ -}}; +#define NX 8 +#define CHUNK_NX 4 + /*------------------------------------------------------------------------- * Function: test_direct_chunk_write * * Purpose: Test the basic functionality of H5DOwrite_chunk * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Raymond Lu - * 30 November 2012 + * Return: Success: An identifer for the dataset used in the tests + * Failure: H5I_INVALID_HID * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_FILTER_DEFLATE -static int -test_direct_chunk_write (hid_t file) +static hid_t +create_dataset(hid_t fid) { - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int ret; - int data[NX][NY]; - int i, j, n; - - unsigned filter_mask = 0; - int direct_buf[CHUNK_NX][CHUNK_NY]; - int check_chunk[CHUNK_NX][CHUNK_NY]; - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - - const Bytef *z_src = (const Bytef*)(direct_buf); - Bytef *z_dst = NULL; /*destination buffer */ - uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); - uLong z_src_nbytes = (uLong)buf_size; - int aggression = 9; /* Compression aggression setting */ - void *outbuf = NULL; /* Pointer to new buffer */ - - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ - - TESTING("basic functionality of H5DOwrite_chunk"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties, i.e. enable chunking and compression - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - - if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME1, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - /* Initialize the dataset */ - for(i = n = 0; i < NX; i++) - for(j = 0; j < NY; j++) - data[i][j] = n++; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* - * Write the data for the dataset. It should stay in the chunk cache. - * It will be evicted from the cache by the H5DOwrite_chunk calls. - */ - if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - dxpl, data)) < 0) - goto error; - - /* Initialize data for one chunk */ - for(i = n = 0; i < CHUNK_NX; i++) - for(j = 0; j < CHUNK_NY; j++) - direct_buf[i][j] = n++; - - /* Allocate output (compressed) buffer */ - outbuf = HDmalloc(z_dst_nbytes); - z_dst = (Bytef *)outbuf; - - /* Perform compression from the source to the destination buffer */ - ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); - - /* Check for various zlib errors */ - if(Z_BUF_ERROR == ret) { - HDfprintf(stderr, "overflow"); - goto error; - } else if(Z_MEM_ERROR == ret) { - HDfprintf(stderr, "deflate memory error"); - goto error; - } else if(Z_OK != ret) { - HDfprintf(stderr, "other deflate error"); - goto error; - } - - /* Write the compressed chunk data repeatedly to cover all the chunks in the - * dataset, using the direct writing function. */ - for(i=0; i0) { - *int_ptr++ -= (int)ADD_ON; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end if */ - else { /* write */ - /* Add the "add on" value to all the data values */ - while(buf_left>0) { - *int_ptr++ += (int)ADD_ON; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end else */ - - return nbytes; -} /* filter_bogus1() */ - -/*------------------------------------------------------------------------- - * Function: filter_bogus2 - * - * Purpose: A bogus filter that multiplies the original value by FACTOR. - * - * Return: Success: Data chunk size - * - * Programmer: Raymond Lu - * 30 November 2012 - *------------------------------------------------------------------------- - */ -static size_t -filter_bogus2(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, - const unsigned int H5_ATTR_UNUSED *cd_values, size_t nbytes, - size_t *buf_size, void **buf) -{ - int *int_ptr=(int *)*buf; /* Pointer to the data values */ - ssize_t buf_left=(ssize_t)*buf_size; /* Amount of data buffer left to process */ - - if(flags & H5Z_FLAG_REVERSE) { /* read */ - /* Substract the "add on" value to all the data values */ - while(buf_left>0) { - *int_ptr++ /= (int)FACTOR; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end if */ - else { /* write */ - /* Add the "add on" value to all the data values */ - while(buf_left>0) { - *int_ptr++ *= (int)FACTOR; - buf_left -= (ssize_t)sizeof(int); - } /* end while */ - } /* end else */ - - return nbytes; -} /* filter_bogus2() */ +} /* test_direct_chunk_write() */ + /*------------------------------------------------------------------------- - * Function: test_skip_compress_write2 + * Function: test_direct_chunk_read * - * Purpose: Test skipping compression filter when there are three filters - * for the dataset + * Purpose: Test the basic functionality of H5DOread_chunk * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Raymond Lu - * 30 November 2012 + * Return: Success: 0 + * Failure: 1 * *------------------------------------------------------------------------- */ static int -test_skip_compress_write2(hid_t file) +test_direct_chunk_read(hid_t did) { - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int i, j, n; - - unsigned filter_mask = 0; /* orig filter mask */ - int origin_direct_buf[CHUNK_NX][CHUNK_NY]; - int direct_buf[CHUNK_NX][CHUNK_NY]; - int check_chunk[CHUNK_NX][CHUNK_NY]; - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - int aggression = 9; /* Compression aggression setting */ + hid_t mem_sid = H5I_INVALID_HID; + hid_t file_sid = H5I_INVALID_HID; + hsize_t dims[1] = {NX}; + hsize_t chunk_dims[1] = {CHUNK_NX}; - unsigned read_filter_mask = 0; /* filter mask after direct read */ - int read_direct_buf[CHUNK_NX][CHUNK_NY]; - hsize_t read_buf_size = 0; /* buf size */ + unsigned filter_mask; + int chunk_data[CHUNK_NX]; /* Chunk read with H5DOread_chunk */ + int check[CHUNK_NX]; /* Chunk read with H5Dread */ + hsize_t offset[1]; - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ + hsize_t start[1]; /* Start of hyperslab */ + hsize_t stride[1]; /* Stride of hyperslab */ + hsize_t count[1]; /* Block count */ + hsize_t block[1]; /* Block sizes */ - TESTING("skipping compression filters but keep two other filters"); + int i,j; - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties, i.e. enable chunking and compression. - * The order of filters is bogus 1 + deflate + bogus 2. - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - - /* Register and enable first bogus filter */ - if(H5Zregister (H5Z_BOGUS1) < 0) - goto error; - - if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS1, 0, (size_t)0, NULL) < 0) - goto error; - - /* Enable compression filter */ - if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) - goto error; - - /* Register and enable second bogus filter */ - if(H5Zregister (H5Z_BOGUS2) < 0) - goto error; - - if(H5Pset_filter(cparms, H5Z_FILTER_BOGUS2, 0, (size_t)0, NULL) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME3, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Initialize data for one chunk. Apply operations of two bogus filters to the chunk */ - for(i = n = 0; i < CHUNK_NX; i++) - for(j = 0; j < CHUNK_NY; j++) { - origin_direct_buf[i][j] = n++; - direct_buf[i][j] = (origin_direct_buf[i][j] + ADD_ON) * FACTOR; - } - - /* write the uncompressed chunk data repeatedly to dataset, using the direct writing function. - * Indicate skipping the compression filter but keep the other two bogus filters */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - - /* compression filter is the middle one to be skipped */ - filter_mask = 0x00000002; - - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) - goto error; - - if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) - goto error; - - if(H5Dclose(dataset) < 0) - goto error; - - if((dataset = H5Dopen2(file, DATASETNAME3, H5P_DEFAULT)) < 0) - goto error; + TESTING("H5DOread_chunk wrapper"); - /* - * Select hyperslab for one chunk in the file - */ - start[0] = CHUNK_NX; start[1] = CHUNK_NY; - stride[0] = 1; stride[1] = 1; - count[0] = 1; count[1] = 1; - block[0] = CHUNK_NX; block[1] = CHUNK_NY; - if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) - goto error; - - /* Read the chunk back */ - if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) - goto error; - - /* Check that the values read are the same as the values written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if(origin_direct_buf[i][j] != check_chunk[i][j]) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" origin_direct_buf=%d, check_chunk=%d\n", origin_direct_buf[i][j], check_chunk[i][j]); - goto error; - } - } - } - - /* Query chunk storage size */ - if((status = H5Dget_chunk_storage_size(dataset, offset, &read_buf_size)) < 0) - goto error; - if(read_buf_size != buf_size) - goto error; + /* Create dataspaces for reading */ + if ((mem_sid = H5Screate_simple(1, chunk_dims, NULL)) < 0) + TEST_ERROR + if ((file_sid = H5Screate_simple(1, dims, NULL)) < 0) + TEST_ERROR - /* Read the raw chunk back */ - HDmemset(&read_direct_buf, 0, sizeof(read_direct_buf)); - if((status = H5DOread_chunk(dataset, H5P_DEFAULT, offset, &read_filter_mask, read_direct_buf)) < 0) - goto error; - if(read_filter_mask != filter_mask) - goto error; - - /* Check that the direct chunk read is the same as the chunk written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if(direct_buf[i][j] != read_direct_buf[i][j]) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" direct_buf=%d, read_direct_buf=%d\n", direct_buf[i][j], read_direct_buf[i][j]); - goto error; - } - } + /* For each chunk in the dataset, compare the result of H5Dread and H5DOread_chunk. */ + for (i = 0; i < NX/CHUNK_NX; i++) { + + /* Select hyperslab for one chunk in the file */ + start[0] = (hsize_t)i * CHUNK_NX; + stride[0] = 1; + count[0] = 1; + block[0] = CHUNK_NX; + + /* Hyperslab selection equals single chunk */ + if (H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, stride, count, block) < 0) + TEST_ERROR + + /* Read the chunk back */ + if (H5Dread(did, H5T_NATIVE_INT, mem_sid, file_sid, H5P_DEFAULT, check) < 0) + TEST_ERROR + + /* Read the raw chunk back */ + HDmemset(chunk_data, 0, CHUNK_NX * sizeof(int)); + filter_mask = UINT_MAX; + offset[0] = (hsize_t)i * CHUNK_NX; + if (H5DOread_chunk(did, H5P_DEFAULT, offset, &filter_mask, chunk_data) < 0) + TEST_ERROR + + /* Check filter mask return value */ + if (filter_mask != 0) + TEST_ERROR + + /* Check that the values are correct */ + for (j = 0; j < CHUNK_NX; j++) + if (chunk_data[i] != check[i]) + TEST_ERROR } - /* - * Close/release resources. - */ - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); + /* Close */ + if (H5Sclose(mem_sid) < 0) + TEST_ERROR + if (H5Sclose(file_sid) < 0) + TEST_ERROR PASSED(); return 0; error: H5E_BEGIN_TRY { - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); + H5Sclose(mem_sid); + H5Sclose(file_sid); } H5E_END_TRY; H5_FAILED(); return 1; -} /* test_skip_compress_write2() */ +} /* test_direct_chunk_read() */ + /*------------------------------------------------------------------------- - * Function: test_data_conv - * - * Purpose: Test data conversion + * Function: Main function * - * Return: Success: 0 - * Failure: 1 + * Purpose: Test direct chunk write function H5DOwrite_chunk and + * chunk direct read function H5DOread_chunk * - * Programmer: Raymond Lu - * 30 November 2012 + * Return: Success: 0 + * Failure: 1 * *------------------------------------------------------------------------- */ -static int -test_data_conv(hid_t file) +int main( void ) { - typedef struct { - int a, b, c[4], d, e; - } src_type_t; - typedef struct { - int a, c[4], e; - } dst_type_t; - - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int i, j, n; - const hsize_t four = 4; - hid_t st=-1, dt=-1; - hid_t array_dt; - - unsigned filter_mask = 0; - src_type_t direct_buf[CHUNK_NX][CHUNK_NY]; - dst_type_t check_chunk[CHUNK_NX][CHUNK_NY]; - src_type_t read_chunk[CHUNK_NX][CHUNK_NY]; /* For H5DOread_chunk */ - - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(src_type_t); - - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ + hid_t fid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + int nerrors = 0; - TESTING("data conversion for H5DOwrite_chunk/H5DOread_chunk"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties, i.e. enable chunking - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + if ((fid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + if ((did = create_dataset(fid)) < 0) goto error; - /* Build hdf5 datatypes */ - array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); - if((st = H5Tcreate(H5T_COMPOUND, sizeof(src_type_t))) < 0 || - H5Tinsert(st, "a", HOFFSET(src_type_t, a), H5T_NATIVE_INT) < 0 || - H5Tinsert(st, "b", HOFFSET(src_type_t, b), H5T_NATIVE_INT) < 0 || - H5Tinsert(st, "c", HOFFSET(src_type_t, c), array_dt) < 0 || - H5Tinsert(st, "d", HOFFSET(src_type_t, d), H5T_NATIVE_INT) < 0 || - H5Tinsert(st, "e", HOFFSET(src_type_t, e), H5T_NATIVE_INT) < 0) - goto error; + nerrors += test_direct_chunk_write(did); + nerrors += test_direct_chunk_read(did); - if(H5Tclose(array_dt) < 0) + if (H5Dclose(did) < 0) goto error; - - array_dt = H5Tarray_create2(H5T_NATIVE_INT, 1, &four); - if((dt = H5Tcreate(H5T_COMPOUND, sizeof(dst_type_t))) < 0 || - H5Tinsert(dt, "a", HOFFSET(dst_type_t, a), H5T_NATIVE_INT) < 0 || - H5Tinsert(dt, "c", HOFFSET(dst_type_t, c), array_dt) < 0 || - H5Tinsert(dt, "e", HOFFSET(dst_type_t, e), H5T_NATIVE_INT) < 0) - goto error; - - if(H5Tclose(array_dt) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME4, st, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Initialize data for one chunk */ - for(i = n = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - (direct_buf[i][j]).a = i*j+0; - (direct_buf[i][j]).b = i*j+1; - (direct_buf[i][j]).c[0] = i*j+2; - (direct_buf[i][j]).c[1] = i*j+3; - (direct_buf[i][j]).c[2] = i*j+4; - (direct_buf[i][j]).c[3] = i*j+5; - (direct_buf[i][j]).d = i*j+6; - (direct_buf[i][j]).e = i*j+7; - } - } - - /* write the chunk data to dataset, using the direct writing function. - * There should be no data conversion involved. */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) < 0) - goto error; - - if(H5Fflush(dataset, H5F_SCOPE_LOCAL) < 0) - goto error; - - if(H5Dclose(dataset) < 0) - goto error; - - if((dataset = H5Dopen2(file, DATASETNAME4, H5P_DEFAULT)) < 0) - goto error; - - /* Use H5DOread_chunk() to read the uncompressed data */ - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, read_chunk)) < 0) - goto error; - - /* Check that the values read are the same as the values written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if ((direct_buf[i][j]).a != (read_chunk[i][j]).a || - (direct_buf[i][j]).b != (read_chunk[i][j]).b || - (direct_buf[i][j]).c[0] != (read_chunk[i][j]).c[0] || - (direct_buf[i][j]).c[1] != (read_chunk[i][j]).c[1] || - (direct_buf[i][j]).c[2] != (read_chunk[i][j]).c[2] || - (direct_buf[i][j]).c[3] != (read_chunk[i][j]).c[3] || - (direct_buf[i][j]).d != (read_chunk[i][j]).d || - (direct_buf[i][j]).e != (read_chunk[i][j]).e) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], - (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); - HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - (read_chunk[i][j]).a, (read_chunk[i][j]).b, (read_chunk[i][j]).c[0], (read_chunk[i][j]).c[1], - (read_chunk[i][j]).c[2], (read_chunk[i][j]).c[3], (read_chunk[i][j]).d, (read_chunk[i][j]).e); - - goto error; - } - } - } - - /* - * Select hyperslab for the chunk just written in the file - */ - start[0] = CHUNK_NX; start[1] = CHUNK_NY; - stride[0] = 1; stride[1] = 1; - count[0] = 1; count[1] = 1; - block[0] = CHUNK_NX; block[1] = CHUNK_NY; - if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0) - goto error; - - /* Read the chunk back. Data should be converted */ - if((status = H5Dread(dataset, dt, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0) - goto error; - - /* Check that the values read are the same as the values written */ - for(i = 0; i < CHUNK_NX; i++) { - for(j = 0; j < CHUNK_NY; j++) { - if ((direct_buf[i][j]).a != (check_chunk[i][j]).a || - (direct_buf[i][j]).c[0] != (check_chunk[i][j]).c[0] || - (direct_buf[i][j]).c[1] != (check_chunk[i][j]).c[1] || - (direct_buf[i][j]).c[2] != (check_chunk[i][j]).c[2] || - (direct_buf[i][j]).c[3] != (check_chunk[i][j]).c[3] || - (direct_buf[i][j]).e != (check_chunk[i][j]).e) { - HDprintf(" 1. Read different values than written."); - HDprintf(" At index %d,%d\n", i, j); - HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - (direct_buf[i][j]).a, (direct_buf[i][j]).b, (direct_buf[i][j]).c[0], (direct_buf[i][j]).c[1], - (direct_buf[i][j]).c[2], (direct_buf[i][j]).c[3], (direct_buf[i][j]).d, (direct_buf[i][j]).e); - HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n", - (check_chunk[i][j]).a, (check_chunk[i][j]).c[0], (check_chunk[i][j]).c[1], (check_chunk[i][j]).c[2], - (check_chunk[i][j]).c[3], (check_chunk[i][j]).e); - - goto error; - } - } - } - - /* - * Close/release resources. - */ - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - H5Tclose(st); - H5Tclose(dt); - - PASSED(); - return 0; - -error: - H5E_BEGIN_TRY { - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - H5Tclose(st); - H5Tclose(dt); - } H5E_END_TRY; - - H5_FAILED(); - return 1; -} /* test_data_conv() */ - -/*------------------------------------------------------------------------- - * Function: test_invalid_parameters - * - * Purpose: Test invalid parameters for H5DOwrite_chunk and H5DOread_chunk - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Raymond Lu - * 30 November 2012 - * - *------------------------------------------------------------------------- - */ -static int -test_invalid_parameters(hid_t file) -{ - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; - herr_t status; - int i, j, n; - - unsigned filter_mask = 0; - int direct_buf[CHUNK_NX][CHUNK_NY]; - hsize_t offset[2] = {0, 0}; - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - int aggression = 9; /* Compression aggression setting */ - - hsize_t chunk_nbytes; /* Chunk size */ - - TESTING("invalid parameters for H5DOwrite_chunk/H5DOread_chunk"); - - /* - * Create the data space with unlimited dimensions. - */ - if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) - goto error; - - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* - * Modify dataset creation properties - */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - - /* - * Create a new contiguous dataset to verify H5DOwrite_chunk/H5DOread_chunk doesn't work - */ - if((dataset = H5Dcreate2(file, DATASETNAME5, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Initialize data for one chunk */ - for(i = n = 0; i < CHUNK_NX; i++) - for(j = 0; j < CHUNK_NY; j++) { - direct_buf[i][j] = n++; - } - - /* Try to write the chunk data to contiguous dataset. It should fail */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Try to get chunk size for a contiguous dataset. It should fail */ - H5E_BEGIN_TRY { - if((status = H5Dget_chunk_storage_size(dataset, offset, &chunk_nbytes)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Try to H5DOread_chunk from the contiguous dataset. It should fail */ - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - if(H5Dclose(dataset) < 0) - goto error; - - - /* Create a chunked dataset with compression filter */ - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - - if((status = H5Pset_deflate( cparms, (unsigned ) aggression)) < 0) - goto error; - - /* - * Create a new dataset within the file using cparms - * creation properties. - */ - if((dataset = H5Dcreate2(file, DATASETNAME6, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, H5P_DEFAULT)) < 0) - goto error; - - /* Check invalid dataset ID for H5DOwrite_chunk and H5DOread_chunk */ - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk((hid_t)-1, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk((hid_t)-1, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid DXPL ID for H5DOwrite_chunk and H5DOread_chunk */ - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, (hid_t)-1, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, (hid_t)-1, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid OFFSET for H5DOwrite_chunk and H5DOread_chunk */ - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, NULL, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, NULL, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check when OFFSET is out of dataset range for H5DOwrite_chunk and H5DOread_chunk */ - offset[0] = NX + 1; - offset[1] = NY; - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check when OFFSET is not on chunk boundary for H5DOwrite_chunk and H5DOread_chunk */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY + 1; - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid buffer size for H5DOwrite_chunk only */ - offset[0] = CHUNK_NX; - offset[1] = CHUNK_NY; - buf_size = 0; - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, direct_buf)) != FAIL) - goto error; - } H5E_END_TRY; - - /* Check invalid data buffer for H5DOwrite_chunk and H5DOread_chunk */ - buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - H5E_BEGIN_TRY { - if((status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, buf_size, NULL)) != FAIL) - goto error; - } H5E_END_TRY; - - H5E_BEGIN_TRY { - if((status = H5DOread_chunk(dataset, dxpl, offset, &filter_mask, NULL)) != FAIL) - goto error; - } H5E_END_TRY; - - if(H5Dclose(dataset) < 0) - goto error; - - /* - * Close/release resources. - */ - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - - PASSED(); - return 0; - -error: - H5E_BEGIN_TRY { - H5Dclose(dataset); - H5Sclose(mem_space); - H5Sclose(dataspace); - H5Pclose(cparms); - H5Pclose(dxpl); - } H5E_END_TRY; - - H5_FAILED(); - return 1; -} /* test_invalid_parameters() */ - -/*------------------------------------------------------------------------- - * Function: test_direct_chunk_read_no_cache - * - * Purpose: Test the basic functionality of H5DOread_chunk with the - * chunk cache diabled. - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Matthew Strong (GE Healthcare) - * 14 February 2016 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_FILTER_DEFLATE -static int -test_direct_chunk_read_no_cache (hid_t file) -{ - hid_t dataspace = -1, dataset = -1; - hid_t mem_space = -1; - hid_t cparms = -1, dxpl = -1, dapl = -1; - hsize_t dims[2] = {NX, NY}; - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; - herr_t status; /* status from H5 function calls */ - int ret; /* deflate return status */ - int data[NX][NY]; - int i, j, k, l, n; /* local index variables */ - - unsigned filter_mask = 0; /* filter mask returned from H5DOread_chunk */ - int direct_buf[CHUNK_NX][CHUNK_NY]; /* chunk read with H5Dread and manually decompressed */ - int check_chunk[CHUNK_NX][CHUNK_NY]; /* chunk read with H5Dread */ - hsize_t offset[2]; /* chunk offset used for H5DOread_chunk */ - size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); - - Bytef *z_src = NULL; /* source buffer */ - uLongf z_src_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); - Bytef *z_dst = (Bytef*)(direct_buf); - uLong z_dst_nbytes = (uLong)buf_size; - int aggression = 9; /* Compression aggression setting */ - void *outbuf = NULL; /* Pointer to new buffer */ - - hsize_t start[2]; /* Start of hyperslab */ - hsize_t stride[2]; /* Stride of hyperslab */ - hsize_t count[2]; /* Block count */ - hsize_t block[2]; /* Block sizes */ - - TESTING("basic functionality of H5DOread_chunk (chunk cache disabled)"); - - /* Create the data space with unlimited dimensions. */ - if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) - goto error; - if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) - goto error; - - /* Modify dataset creation properties, i.e. enable chunking and compression */ - if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) - goto error; - if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) - goto error; - if((status = H5Pset_deflate( cparms, (unsigned) aggression)) < 0) - goto error; - if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) - goto error; - - /* Disable chunk cache by setting number of slots to 0 */ - if((status = H5Pset_chunk_cache(dapl, 0, H5D_CHUNK_CACHE_NBYTES_DEFAULT, H5D_CHUNK_CACHE_W0_DEFAULT)) < 0) - goto error; - - /* Create a new dataset within the file using cparms creation properties. */ - if((dataset = H5Dcreate2(file, DATASETNAME8, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, - cparms, dapl)) < 0) - goto error; - - /* Initialize the dataset */ - for(i = n = 0; i < NX; i++) - for(j = 0; j < NY; j++) - data[i][j] = n++; - - if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) - goto error; - - /* Write the data for the dataset. - * Data will skip chunk cache and go directly to disk. */ - if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - dxpl, data)) < 0) - goto error; - - /* Allocate output (compressed) buffer */ - outbuf = HDmalloc(z_src_nbytes); - z_src = (Bytef *)outbuf; - - /* For each chunk in the dataset, compare the result of H5Dread and H5DOread_chunk. */ - for(i=0; i Date: Fri, 4 May 2018 14:16:56 -0700 Subject: Updated commenting in the H5DO compat test. --- hl/test/test_h5do_compat.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/hl/test/test_h5do_compat.c b/hl/test/test_h5do_compat.c index 486f9da..dc30414 100644 --- a/hl/test/test_h5do_compat.c +++ b/hl/test/test_h5do_compat.c @@ -1,23 +1,21 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -* Copyright by The HDF Group. * -* Copyright by the Board of Trustees of the University of Illinois. * -* All rights reserved. * -* * -* This file is part of HDF5. The full HDF5 copyright notice, including * -* terms governing use, modification, and redistribution, is contained in * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * * the COPYING file, which can be found at the root of the source code * * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * -* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "h5hltest.h" #include "H5DOpublic.h" -/* This test is a direct copy of the direct_chunk.c test from test/ but - * with the H5Dread/write_chunk calls replaced with H5DOread/write_chunk. - * This ensures that older code which uses the H5DO calls can still - * work with the newer library. +/* This test is a minimal test to ensure that the H5DO compatibility wrappers + * work correctly. */ #define FILE_NAME "h5do_compat.h5" -- cgit v0.12 From 1c11ddec6d6d9aa39ea7d56a183e7a096bc9076f Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Fri, 4 May 2018 16:43:06 -0700 Subject: Added deprecated symbol wrappers for the H5DOwrite/read_chunk wrappers. --- hl/src/H5DO.c | 3 +++ hl/src/H5DOpublic.h | 8 ++++++++ hl/test/test_h5do_compat.c | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/hl/src/H5DO.c b/hl/src/H5DO.c index 3caa32b..057c43b 100644 --- a/hl/src/H5DO.c +++ b/hl/src/H5DO.c @@ -17,6 +17,7 @@ /* public LT prototypes */ #include "H5DOpublic.h" +#ifndef H5_NO_DEPRECATED_SYMBOLS /*------------------------------------------------------------------------- * Function: H5DOwrite_chunk @@ -68,6 +69,8 @@ H5DOread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *fi return SUCCEED; } /* end H5DOread_chunk() */ +#endif /* H5_NO_DEPRECATED_SYMBOLS */ + /*------------------------------------------------------------------------- * Function: H5DOappend() diff --git a/hl/src/H5DOpublic.h b/hl/src/H5DOpublic.h index 0271938..e09ebca 100644 --- a/hl/src/H5DOpublic.h +++ b/hl/src/H5DOpublic.h @@ -28,12 +28,20 @@ extern "C" { H5_HLDLL herr_t H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, hid_t memtype, const void *buf); +/* Symbols defined for compatibility with previous versions of the HDF5 API. + * + * Use of these symbols is deprecated. + */ +#ifndef H5_NO_DEPRECATED_SYMBOLS + /* Compatibility wrappers for functionality moved to H5D */ H5_HLDLL herr_t H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, size_t data_size, const void *buf); H5_HLDLL herr_t H5DOread_chunk(hid_t dset_id, hid_t dxpl_id, const hsize_t *offset, uint32_t *filters /*out*/, void *buf /*out*/); +#endif /* H5_NO_DEPRECATED_SYMBOLS */ + #ifdef __cplusplus } #endif diff --git a/hl/test/test_h5do_compat.c b/hl/test/test_h5do_compat.c index dc30414..4df5eef 100644 --- a/hl/test/test_h5do_compat.c +++ b/hl/test/test_h5do_compat.c @@ -18,6 +18,8 @@ * work correctly. */ +#ifndef H5_NO_DEPRECATED_SYMBOLS + #define FILE_NAME "h5do_compat.h5" #define DATASET_NAME "direct_chunk_io" @@ -228,9 +230,11 @@ error: return 1; } /* test_direct_chunk_read() */ +#endif /* H5_NO_DEPRECATED_SYMBOLS */ + /*------------------------------------------------------------------------- - * Function: Main function + * Function: main * * Purpose: Test direct chunk write function H5DOwrite_chunk and * chunk direct read function H5DOread_chunk @@ -242,6 +246,14 @@ error: */ int main( void ) { +#ifdef H5_NO_DEPRECATED_SYMBOLS + + HDputs("Direct chunk read/write wrapper tests SKIPPED."); + HDputs("(Backward compatibility not configured)"); + return EXIT_SUCCESS; + +#else + hid_t fid = H5I_INVALID_HID; hid_t did = H5I_INVALID_HID; int nerrors = 0; @@ -270,4 +282,5 @@ int main( void ) error: HDputs("*** TESTS FAILED ***"); return EXIT_FAILURE; +#endif /* H5_NO_DEPRECATED_SYMBOLS */ } /* end main() */ -- cgit v0.12 From 1da9c5545c013ebc540ba3044810889d4acfa5be Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 14 May 2018 17:50:11 -0700 Subject: Restored some unused #defines to the deprecated section of H5Dpublic.h. --- src/H5Dpublic.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 6cec3f5..a1ccda0 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -178,7 +178,19 @@ H5_DLL herr_t H5Dget_chunk_index_type(hid_t did, H5D_chunk_index_t *idx_type); /* Macros */ #define H5D_CHUNK_BTREE H5D_CHUNK_IDX_BTREE - +/* Formerly used to support the H5DOread/write_chunk() API calls. + * These symbols are no longer used in the library. + */ +/* Property names for H5DOwrite_chunk */ +#define H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME "direct_chunk_flag" +#define H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME "direct_chunk_filters" +#define H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME "direct_chunk_offset" +#define H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME "direct_chunk_datasize" +/* Property names for H5DOread_chunk */ +#define H5D_XFER_DIRECT_CHUNK_READ_FLAG_NAME "direct_chunk_read_flag" +#define H5D_XFER_DIRECT_CHUNK_READ_OFFSET_NAME "direct_chunk_read_offset" +#define H5D_XFER_DIRECT_CHUNK_READ_FILTERS_NAME "direct_chunk_read_filters" + /* Typedefs */ -- cgit v0.12