From e9a9c23a6dbc1c3aa05a4dcb25702c49533517db Mon Sep 17 00:00:00 2001 From: Raymond Lu Date: Wed, 7 Nov 2012 12:11:22 -0500 Subject: [svn-r23018] I forgot to add these new files in previous commit. --- hl/src/H5DO.c | 111 +++++++++++++++++ hl/src/H5DOprivate.h | 37 ++++++ hl/src/H5DOpublic.h | 42 +++++++ hl/test/test_dset_opt.c | 318 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 508 insertions(+) create mode 100644 hl/src/H5DO.c create mode 100644 hl/src/H5DOprivate.h create mode 100644 hl/src/H5DOpublic.h create mode 100644 hl/test/test_dset_opt.c diff --git a/hl/src/H5DO.c b/hl/src/H5DO.c new file mode 100644 index 0000000..5b8041e --- /dev/null +++ b/hl/src/H5DO.c @@ -0,0 +1,111 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Copyright by The HDF Group. * +* Copyright by the Board of Trustees of the University of Illinois. * +* All rights reserved. * +* * +* This file is part of HDF5. The full HDF5 copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the files COPYING and Copyright.html. COPYING can be found at the root * +* of the source code distribution tree; Copyright.html can be found at the * +* root level of an installed copy of the electronic HDF5 document set and * +* is linked from the top-level documents page. It can also be found at * +* http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * +* access to either file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include +#include + +#include "H5DOprivate.h" + +/*------------------------------------------------------------------------- + * Function: H5DOwrite_chunk + * + * Purpose: Writes an entire chunk to the file directly. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 30 July 2012 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, hsize_t *offset, + size_t data_size, const void *buf) +{ + htri_t created_dxpl = FALSE; + + if(dset_id < 0) + goto error; + + if(!buf) + goto error; + + if(!offset) + goto error; + + if(!data_size) + goto error; + + if(H5P_DEFAULT == dxpl_id) { + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + created_dxpl = TRUE; + } + + if(H5DO_write_chunk(dset_id, dxpl_id, filters, offset, data_size, buf) < 0) + goto error; + + if(created_dxpl) { + if(H5Pclose(dxpl_id) < 0) + goto error; + } + +error: + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: H5DO_write_chunk + * + * Purpose: Private function for H5DOwrite_chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 30 July 2012 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5DO_write_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, hsize_t *offset, + size_t data_size, const void *buf) +{ + htri_t do_direct_write = TRUE; + + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) + goto error; + + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, &filters) < 0) + goto error; + + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, &offset) < 0) + goto error; + + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, &data_size) < 0) + goto error; + + if(H5Dwrite(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) + goto error; + + do_direct_write = FALSE; + if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0) + goto error; + +error: + return FAIL; +} diff --git a/hl/src/H5DOprivate.h b/hl/src/H5DOprivate.h new file mode 100644 index 0000000..8617567 --- /dev/null +++ b/hl/src/H5DOprivate.h @@ -0,0 +1,37 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef _H5DOprivate_H +#define _H5DOprivate_H + +/* High-level library internal header file */ +#include "H5HLprivate2.h" + +/* public LT prototypes */ +#include "H5DOpublic.h" + +/*------------------------------------------------------------------------- + * Private functions + *------------------------------------------------------------------------- + */ + +H5_HLDLL herr_t H5DO_write_chunk(hid_t dset_id, + hid_t dxpl_id, + uint32_t filters, + hsize_t *offset, + size_t data_size, + const void *buf); + +#endif diff --git a/hl/src/H5DOpublic.h b/hl/src/H5DOpublic.h new file mode 100644 index 0000000..b20de56 --- /dev/null +++ b/hl/src/H5DOpublic.h @@ -0,0 +1,42 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef _H5DOpublic_H +#define _H5DOpublic_H + +#ifdef __cplusplus +extern "C" { +#endif + +/*------------------------------------------------------------------------- + * + * Direct chunk write function + * + *------------------------------------------------------------------------- + */ + +H5_HLDLL herr_t H5DOwrite_chunk(hid_t dset_id, + hid_t dxpl_id, + uint32_t filters, + hsize_t *offset, + size_t data_size, + const void *buf); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/hl/test/test_dset_opt.c b/hl/test/test_dset_opt.c new file mode 100644 index 0000000..35dc73c --- /dev/null +++ b/hl/test/test_dset_opt.c @@ -0,0 +1,318 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Copyright by The HDF Group. * +* Copyright by the Board of Trustees of the University of Illinois. * +* All rights reserved. * +* * +* This file is part of HDF5. The full HDF5 copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the files COPYING and Copyright.html. COPYING can be found at the root * +* of the source code distribution tree; Copyright.html can be found at the * +* root level of an installed copy of the electronic HDF5 document set and * +* is linked from the top-level documents page. It can also be found at * +* http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * +* access to either file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include +#include "h5hltest.h" +#include "H5srcdir.h" +#include "H5DOpublic.h" +#include +#include + +#define FILE_NAME5 "test_dectris.h5" + +#define DATASETNAME "Array" +#define RANK 2 +#define NX 16 +#define NY 16 +#define CHUNK_NX 4 +#define CHUNK_NY 4 + +#define DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s))*1.001)+12) + +static int +test_direct_chunk_write (void) +{ + char filename[1024]; + hid_t file; /* handles */ + hid_t fapl; + hid_t dataspace, dataset; + hid_t mem_space; + hid_t cparms, dxpl; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + herr_t status; + int ret; + int data[NX][NY]; + int check[NX][NY]; + int i, j, n; + + unsigned filter_mask = 0; + int direct_buf[CHUNK_NX][CHUNK_NY]; + int check_chunk[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {0, 0}; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + + const Bytef *z_src = (const Bytef*)(direct_buf); + Bytef *z_dst; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + uLong z_src_nbytes = (uLong)buf_size; + int aggression = 9; /* Compression aggression setting */ + void *outbuf = NULL; /* Pointer to new buffer */ + + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + + TESTING("H5DOwrite_chunk"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + goto error; + + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + goto error; + + /* + * Create a new file. If file exists its contents will be overwritten. + */ + if((file = H5Fcreate(FILE_NAME5, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + + if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0) + goto error; + + if((status = H5Pset_deflate( cparms, aggression)) < 0) + goto error; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, DATASETNAME, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + goto error; + + /* Initialize the dataset */ + for(i = n = 0; i < NX; i++) + for(j = 0; j < NY; j++) + data[i][j] = n++; + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + + /* + * Write the data for the dataset. It should stay in the chunk cache. + * It will be evicted from the cache by the H5DOwrite_chunk calls. + */ + if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + dxpl, data)) < 0) + goto error; + + /* Initialize data for one chunk */ + for(i = n = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) + direct_buf[i][j] = n++; + + /* Allocate output (compressed) buffer */ + outbuf = malloc(z_dst_nbytes); + z_dst = (Bytef *)outbuf; + + /* Perform compression from the source to the destination buffer */ + ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == ret) { + fprintf(stderr, "overflow"); + goto error; + } else if(Z_MEM_ERROR == ret) { + fprintf(stderr, "deflate memory error"); + goto error; + } else if(Z_OK != ret) { + fprintf(stderr, "other deflate error"); + goto error; + } + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i