summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRaymond Lu <songyulu@hdfgroup.org>2012-11-07 17:11:22 (GMT)
committerRaymond Lu <songyulu@hdfgroup.org>2012-11-07 17:11:22 (GMT)
commite9a9c23a6dbc1c3aa05a4dcb25702c49533517db (patch)
tree4af113a672241733dccb9dbaa2cf52472779987a
parentb7041a91cd7b322007e87ee3b37729ce0319a56c (diff)
downloadhdf5-e9a9c23a6dbc1c3aa05a4dcb25702c49533517db.zip
hdf5-e9a9c23a6dbc1c3aa05a4dcb25702c49533517db.tar.gz
hdf5-e9a9c23a6dbc1c3aa05a4dcb25702c49533517db.tar.bz2
[svn-r23018] I forgot to add these new files in previous commit.
-rw-r--r--hl/src/H5DO.c111
-rw-r--r--hl/src/H5DOprivate.h37
-rw-r--r--hl/src/H5DOpublic.h42
-rw-r--r--hl/test/test_dset_opt.c318
4 files changed, 508 insertions, 0 deletions
diff --git a/hl/src/H5DO.c b/hl/src/H5DO.c
new file mode 100644
index 0000000..5b8041e
--- /dev/null
+++ b/hl/src/H5DO.c
@@ -0,0 +1,111 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+* Copyright by The HDF Group. *
+* Copyright by the Board of Trustees of the University of Illinois. *
+* All rights reserved. *
+* *
+* This file is part of HDF5. The full HDF5 copyright notice, including *
+* terms governing use, modification, and redistribution, is contained in *
+* the files COPYING and Copyright.html. COPYING can be found at the root *
+* of the source code distribution tree; Copyright.html can be found at the *
+* root level of an installed copy of the electronic HDF5 document set and *
+* is linked from the top-level documents page. It can also be found at *
+* http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+* access to either file, you may request a copy from help@hdfgroup.org. *
+* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdio.h>
+
+#include "H5DOprivate.h"
+
+/*-------------------------------------------------------------------------
+ * Function: H5DOwrite_chunk
+ *
+ * Purpose: Writes an entire chunk to the file directly.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Raymond Lu
+ * 30 July 2012
+ *
+ * Modifications:
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, hsize_t *offset,
+ size_t data_size, const void *buf)
+{
+ htri_t created_dxpl = FALSE;
+
+ if(dset_id < 0)
+ goto error;
+
+ if(!buf)
+ goto error;
+
+ if(!offset)
+ goto error;
+
+ if(!data_size)
+ goto error;
+
+ if(H5P_DEFAULT == dxpl_id) {
+ dxpl_id = H5Pcreate(H5P_DATASET_XFER);
+ created_dxpl = TRUE;
+ }
+
+ if(H5DO_write_chunk(dset_id, dxpl_id, filters, offset, data_size, buf) < 0)
+ goto error;
+
+ if(created_dxpl) {
+ if(H5Pclose(dxpl_id) < 0)
+ goto error;
+ }
+
+error:
+ return FAIL;
+}
+
+/*-------------------------------------------------------------------------
+ * Function: H5DO_write_chunk
+ *
+ * Purpose: Private function for H5DOwrite_chunk
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Raymond Lu
+ * 30 July 2012
+ *
+ * Modifications:
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5DO_write_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, hsize_t *offset,
+ size_t data_size, const void *buf)
+{
+ htri_t do_direct_write = TRUE;
+
+ if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0)
+ goto error;
+
+ if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FILTERS_NAME, &filters) < 0)
+ goto error;
+
+ if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_OFFSET_NAME, &offset) < 0)
+ goto error;
+
+ if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_NAME, &data_size) < 0)
+ goto error;
+
+ if(H5Dwrite(dset_id, 0, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0)
+ goto error;
+
+ do_direct_write = FALSE;
+ if(H5Pset(dxpl_id, H5D_XFER_DIRECT_CHUNK_WRITE_FLAG_NAME, &do_direct_write) < 0)
+ goto error;
+
+error:
+ return FAIL;
+}
diff --git a/hl/src/H5DOprivate.h b/hl/src/H5DOprivate.h
new file mode 100644
index 0000000..8617567
--- /dev/null
+++ b/hl/src/H5DOprivate.h
@@ -0,0 +1,37 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#ifndef _H5DOprivate_H
+#define _H5DOprivate_H
+
+/* High-level library internal header file */
+#include "H5HLprivate2.h"
+
+/* public LT prototypes */
+#include "H5DOpublic.h"
+
+/*-------------------------------------------------------------------------
+ * Private functions
+ *-------------------------------------------------------------------------
+ */
+
+H5_HLDLL herr_t H5DO_write_chunk(hid_t dset_id,
+ hid_t dxpl_id,
+ uint32_t filters,
+ hsize_t *offset,
+ size_t data_size,
+ const void *buf);
+
+#endif
diff --git a/hl/src/H5DOpublic.h b/hl/src/H5DOpublic.h
new file mode 100644
index 0000000..b20de56
--- /dev/null
+++ b/hl/src/H5DOpublic.h
@@ -0,0 +1,42 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#ifndef _H5DOpublic_H
+#define _H5DOpublic_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*-------------------------------------------------------------------------
+ *
+ * Direct chunk write function
+ *
+ *-------------------------------------------------------------------------
+ */
+
+H5_HLDLL herr_t H5DOwrite_chunk(hid_t dset_id,
+ hid_t dxpl_id,
+ uint32_t filters,
+ hsize_t *offset,
+ size_t data_size,
+ const void *buf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
diff --git a/hl/test/test_dset_opt.c b/hl/test/test_dset_opt.c
new file mode 100644
index 0000000..35dc73c
--- /dev/null
+++ b/hl/test/test_dset_opt.c
@@ -0,0 +1,318 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+* Copyright by The HDF Group. *
+* Copyright by the Board of Trustees of the University of Illinois. *
+* All rights reserved. *
+* *
+* This file is part of HDF5. The full HDF5 copyright notice, including *
+* terms governing use, modification, and redistribution, is contained in *
+* the files COPYING and Copyright.html. COPYING can be found at the root *
+* of the source code distribution tree; Copyright.html can be found at the *
+* root level of an installed copy of the electronic HDF5 document set and *
+* is linked from the top-level documents page. It can also be found at *
+* http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+* access to either file, you may request a copy from help@hdfgroup.org. *
+* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#include <stdlib.h>
+#include <string.h>
+#include "h5hltest.h"
+#include "H5srcdir.h"
+#include "H5DOpublic.h"
+#include <zlib.h>
+#include <math.h>
+
+#define FILE_NAME5 "test_dectris.h5"
+
+#define DATASETNAME "Array"
+#define RANK 2
+#define NX 16
+#define NY 16
+#define CHUNK_NX 4
+#define CHUNK_NY 4
+
+#define DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s))*1.001)+12)
+
+static int
+test_direct_chunk_write (void)
+{
+ char filename[1024];
+ hid_t file; /* handles */
+ hid_t fapl;
+ hid_t dataspace, dataset;
+ hid_t mem_space;
+ hid_t cparms, dxpl;
+ hsize_t dims[2] = {NX, NY};
+ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
+ hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY};
+ herr_t status;
+ int ret;
+ int data[NX][NY];
+ int check[NX][NY];
+ int i, j, n;
+
+ unsigned filter_mask = 0;
+ int direct_buf[CHUNK_NX][CHUNK_NY];
+ int check_chunk[CHUNK_NX][CHUNK_NY];
+ hsize_t offset[2] = {0, 0};
+ size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
+
+ const Bytef *z_src = (const Bytef*)(direct_buf);
+ Bytef *z_dst; /*destination buffer */
+ uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size);
+ uLong z_src_nbytes = (uLong)buf_size;
+ int aggression = 9; /* Compression aggression setting */
+ void *outbuf = NULL; /* Pointer to new buffer */
+
+ hsize_t start[2]; /* Start of hyperslab */
+ hsize_t stride[2]; /* Stride of hyperslab */
+ hsize_t count[2]; /* Block count */
+ hsize_t block[2]; /* Block sizes */
+
+ TESTING("H5DOwrite_chunk");
+
+ /*
+ * Create the data space with unlimited dimensions.
+ */
+ if((dataspace = H5Screate_simple(RANK, dims, maxdims)) < 0)
+ goto error;
+
+ if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0)
+ goto error;
+
+ /*
+ * Create a new file. If file exists its contents will be overwritten.
+ */
+ if((file = H5Fcreate(FILE_NAME5, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ goto error;
+
+ /*
+ * Modify dataset creation properties, i.e. enable chunking and compression
+ */
+ if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ goto error;
+
+ if((status = H5Pset_chunk( cparms, RANK, chunk_dims)) < 0)
+ goto error;
+
+ if((status = H5Pset_deflate( cparms, aggression)) < 0)
+ goto error;
+
+ /*
+ * Create a new dataset within the file using cparms
+ * creation properties.
+ */
+ if((dataset = H5Dcreate2(file, DATASETNAME, H5T_NATIVE_INT, dataspace, H5P_DEFAULT,
+ cparms, H5P_DEFAULT)) < 0)
+ goto error;
+
+ /* Initialize the dataset */
+ for(i = n = 0; i < NX; i++)
+ for(j = 0; j < NY; j++)
+ data[i][j] = n++;
+
+ if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0)
+ goto error;
+
+ /*
+ * Write the data for the dataset. It should stay in the chunk cache.
+ * It will be evicted from the cache by the H5DOwrite_chunk calls.
+ */
+ if((status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
+ dxpl, data)) < 0)
+ goto error;
+
+ /* Initialize data for one chunk */
+ for(i = n = 0; i < CHUNK_NX; i++)
+ for(j = 0; j < CHUNK_NY; j++)
+ direct_buf[i][j] = n++;
+
+ /* Allocate output (compressed) buffer */
+ outbuf = malloc(z_dst_nbytes);
+ z_dst = (Bytef *)outbuf;
+
+ /* Perform compression from the source to the destination buffer */
+ ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+
+ /* Check for various zlib errors */
+ if(Z_BUF_ERROR == ret) {
+ fprintf(stderr, "overflow");
+ goto error;
+ } else if(Z_MEM_ERROR == ret) {
+ fprintf(stderr, "deflate memory error");
+ goto error;
+ } else if(Z_OK != ret) {
+ fprintf(stderr, "other deflate error");
+ goto error;
+ }
+
+ /* Write the compressed chunk data repeatedly to cover all the chunks in the
+ * dataset, using the direct writing function. */
+ for(i=0; i<NX/CHUNK_NX; i++) {
+ for(j=0; j<NY/CHUNK_NY; j++) {
+ status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, z_dst_nbytes, outbuf);
+ offset[1] += CHUNK_NY;
+ }
+ offset[0] += CHUNK_NX;
+ offset[1] = 0;
+ }
+
+ if(outbuf)
+ free(outbuf);
+
+ if(H5Dclose(dataset) < 0)
+ goto error;
+
+ if(H5Fclose(file) < 0)
+ goto error;
+
+ /* Reopen the file and dataset */
+ if((file = H5Fopen(FILE_NAME5, H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
+ goto error;
+
+ if((dataset = H5Dopen(file, DATASETNAME, H5P_DEFAULT)) < 0)
+ goto error;
+
+ /*
+ * Select hyperslab for one chunk in the file
+ */
+ start[0] = CHUNK_NX; start[1] = CHUNK_NY;
+ stride[0] = 1; stride[1] = 1;
+ count[0] = 1; count[1] = 1;
+ block[0] = CHUNK_NX; block[1] = CHUNK_NY;
+ if((status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, stride, count, block)) < 0)
+ goto error;
+
+ /* Read the chunk back */
+ if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0)
+ goto error;
+
+ /* Check that the values read are the same as the values written */
+ for(i = 0; i < CHUNK_NX; i++) {
+ for(j = 0; j < CHUNK_NY; j++) {
+ if(direct_buf[i][j] != check_chunk[i][j]) {
+ printf(" 1. Read different values than written.");
+ printf(" At index %d,%d\n", i, j);
+ printf(" direct_buf=%d, check_chunk=%d\n", direct_buf[i][j], check_chunk[i][j]);
+ goto error;
+ }
+ }
+ }
+
+ /* Reinitialize different data for one chunk */
+ for(i = 0; i < CHUNK_NX; i++)
+ for(j = 0; j < CHUNK_NY; j++)
+ direct_buf[i][j] = i + j;
+
+ /* Allocate output (compressed) buffer */
+ outbuf = malloc(z_dst_nbytes);
+ z_dst = (Bytef *)outbuf;
+
+ /* Perform compression from the source to the destination buffer */
+ ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
+
+ /* Check for various zlib errors */
+ if(Z_BUF_ERROR == ret) {
+ fprintf(stderr, "overflow");
+ goto error;
+ } else if(Z_MEM_ERROR == ret) {
+ fprintf(stderr, "deflate memory error");
+ goto error;
+ } else if(Z_OK != ret) {
+ fprintf(stderr, "other deflate error");
+ goto error;
+ }
+
+ /* Rewrite the compressed chunk data repeatedly to cover all the chunks in the
+ * dataset, using the direct writing function. */
+ offset[0] = offset[1] = 0;
+ for(i=0; i<NX/CHUNK_NX; i++) {
+ for(j=0; j<NY/CHUNK_NY; j++) {
+ status = H5DOwrite_chunk(dataset, dxpl, filter_mask, offset, z_dst_nbytes, outbuf);
+ offset[1] += CHUNK_NY;
+ }
+ offset[0] += CHUNK_NX;
+ offset[1] = 0;
+ }
+
+ if(outbuf)
+ free(outbuf);
+
+ if(H5Dclose(dataset) < 0)
+ goto error;
+
+ if(H5Fclose(file) < 0)
+ goto error;
+
+ /* Reopen the file and dataset */
+ if((file = H5Fopen(FILE_NAME5, H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
+ goto error;
+
+ if((dataset = H5Dopen(file, DATASETNAME, H5P_DEFAULT)) < 0)
+ goto error;
+
+ /* Read the chunk back */
+ if((status = H5Dread(dataset, H5T_NATIVE_INT, mem_space, dataspace, H5P_DEFAULT, check_chunk)) < 0)
+ goto error;
+
+ /* Check that the values read are the same as the values written */
+ for(i = 0; i < CHUNK_NX; i++) {
+ for(j = 0; j < CHUNK_NY; j++) {
+ if(direct_buf[i][j] != check_chunk[i][j]) {
+ printf(" 2. Read different values than written.");
+ printf(" At index %d,%d\n", i, j);
+ printf(" direct_buf=%d, check_chunk=%d\n", direct_buf[i][j], check_chunk[i][j]);
+ goto error;
+ }
+ }
+ }
+
+ /*
+ * Close/release resources.
+ */
+ H5Dclose(dataset);
+ H5Sclose(mem_space);
+ H5Sclose(dataspace);
+ H5Pclose(cparms);
+ H5Pclose(dxpl);
+ H5Fclose(file);
+
+ /* h5_cleanup(FILE_NAME5, fapl); */
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY {
+ H5Dclose(dataset);
+ H5Sclose(mem_space);
+ H5Sclose(dataspace);
+ H5Pclose(cparms);
+ H5Pclose(dxpl);
+ H5Fclose(file);
+ } H5E_END_TRY;
+
+ if(outbuf)
+ free(outbuf);
+
+ return 1;
+}
+
+/*-------------------------------------------------------------------------
+* the main program
+*-------------------------------------------------------------------------
+*/
+int main( void )
+{
+ int nerrors=0;
+
+ /* Test direct chunk write */
+ nerrors += test_direct_chunk_write();
+
+ /* check for errors */
+ if (nerrors)
+ goto error;
+
+ return 0;
+
+error:
+ return 1;
+}