From faba610060cb168d97a7b9c01d95688542e2cf28 Mon Sep 17 00:00:00 2001 From: Raymond Lu Date: Wed, 8 Aug 2012 11:08:27 -0500 Subject: [svn-r22641] Dectris project: I revised the code per Quincey's and Neil's comments. I added a performance benchmark program dectris_perf.c in the test/ directory. Tested on koala and jam. --- src/H5Dchunk.c | 28 +-- src/H5Dio.c | 31 ++- src/H5Dprivate.h | 4 +- src/H5Dpublic.h | 4 +- src/H5F.c | 6 +- src/H5FDsec2.c | 2 + src/H5Fpkg.h | 3 +- src/H5Fprivate.h | 2 - src/H5Fpublic.h | 1 + src/H5Fquery.c | 29 --- test/Makefile.am | 4 +- test/Makefile.in | 47 +++-- test/dectris_perf.c | 535 ++++++++++++++++++++++++++++++++++++++++++++++++++++ test/dectris_tst.c | 79 +++++++- 14 files changed, 696 insertions(+), 79 deletions(-) create mode 100644 test/dectris_perf.c diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index e9a50ef..7cde7b6 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -60,8 +60,6 @@ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Vprivate.h" /* Vector and array functions */ -#include "H5Fprivate.h" -#include "H5FDprivate.h" /****************/ @@ -298,8 +296,8 @@ H5FL_BLK_DEFINE_STATIC(chunk); *------------------------------------------------------------------------- */ herr_t -H5D__direct_write(const H5D_t *dset, hid_t dxpl_id, size_t *offset, size_t buf_size, - const void *buf) +H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, unsigned filters, hsize_t *offset, + size_t data_size, const void *buf) { const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ H5D_chunk_ud_t udata; /* User data for querying chunk info */ @@ -337,8 +335,12 @@ H5D__direct_write(const H5D_t *dset, hid_t dxpl_id, size_t *offset, size_t buf_s &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") - /* If the chunk hasn't been allocated on disk, do so now. */ - if(!H5F_addr_defined(udata.addr)) { + udata.filter_mask = filters; + + /* Check if the chunk needs to be 'inserted' (could exist already and + * the 'insert' operation could resize it) + */ + { H5D_chk_idx_info_t idx_info; /* Chunked index info */ /* Compose chunked index info struct */ @@ -349,9 +351,11 @@ H5D__direct_write(const H5D_t *dset, hid_t dxpl_id, size_t *offset, size_t buf_s idx_info.storage = &(dset->shared->layout.storage.u.chunk); /* Set up the size of chunk for user data */ - udata.nbytes = buf_size; + udata.nbytes = data_size; - /* Create the chunk */ + /* Create the chunk it if it doesn't exist, or reallocate the chunk + * if its size changed. + */ if((dset->shared->layout.storage.u.chunk.ops->insert)(&idx_info, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk") @@ -372,10 +376,10 @@ H5D__direct_write(const H5D_t *dset, hid_t dxpl_id, size_t *offset, size_t buf_s HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk") } /* end if */ - /* Write the data to the file driver, instead of H5F_block_write */ - lf = H5F_DRIVER(dset->oloc.file); - if(H5FD_write(lf, dxpl_id, H5FD_MEM_DRAW, udata.addr, buf_size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") + /* Write the data to the file */ + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, data_size, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file") + done: /*FUNC_LEAVE_NOAPI(ret_value)*/ diff --git a/src/H5Dio.c b/src/H5Dio.c index accb948..9576177 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -28,6 +28,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ +#include "H5Sprivate.h" /* Dataspace */ #ifdef H5_HAVE_PARALLEL /* Remove this if H5R_DATASET_REGION is no longer used in this file */ @@ -284,10 +285,13 @@ done: *------------------------------------------------------------------------- */ herr_t -H5PSIdirect_write(hid_t dset_id, hid_t dxpl_id, size_t *offset, size_t buf_size, - const void *buf) +H5PSIdirect_write(hid_t dset_id, hid_t dxpl_id, unsigned filters, hsize_t *offset, + size_t data_size, const void *buf) { H5D_t *dset = NULL; + int ndims; + hsize_t *dims; + int i; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -298,6 +302,9 @@ H5PSIdirect_write(hid_t dset_id, hid_t dxpl_id, size_t *offset, size_t buf_size, if(NULL == dset->oloc.file) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + /* Get the default dataset transfer property list if the user didn't provide one */ if(H5P_DEFAULT == dxpl_id) dxpl_id= H5P_DATASET_XFER_DEFAULT; @@ -308,8 +315,26 @@ H5PSIdirect_write(hid_t dset_id, hid_t dxpl_id, size_t *offset, size_t buf_size, if(!buf) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no output buffer") + ndims = (int)H5S_GET_EXTENT_NDIMS(dset->shared->space); + dims = (hsize_t *)HDmalloc(ndims*sizeof(hsize_t)); + + if(H5S_get_simple_extent_dims(dset->shared->space, dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve dataspace extent dims") + + /* Make sure the offset doesn't exceed the dataset's dimensions */ + for(i=0; i dims[i]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset exceeds dimensions of dataset") + + /* Make sure the offset fall right on a chunk's boundary */ + for(i=0; ishared->layout.u.chunk.dim[i]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "offset doesn't fall on chunks's boundary") + + HDfree(dims); + /* write raw data */ - if(H5D__direct_write(dset, dxpl_id, offset, buf_size, buf) < 0) + if(H5D__chunk_direct_write(dset, dxpl_id, filters, offset, data_size, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data") done: diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index b52800b..342c284 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -176,8 +176,8 @@ H5_DLL herr_t H5D_chunk_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_ad H5_DLL herr_t H5D_btree_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, int fwidth, unsigned ndims); -H5_DLL herr_t H5D__direct_write(const H5D_t *dset, hid_t dxpl_id, size_t *offset, - size_t buf_size, const void *buf); +H5_DLL herr_t H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, unsigned filters, + hsize_t *offset, size_t data_size, const void *buf); #endif /* _H5Dprivate_H */ diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index b072e64..58c2a2b 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -118,8 +118,8 @@ H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf/*out*/); H5_DLL herr_t H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf); -H5_DLL herr_t H5PSIdirect_write(hid_t dset_id, hid_t dxpl_id, size_t *offset, size_t buf_size, - const void *buf); +H5_DLL herr_t H5PSIdirect_write(hid_t dset_id, hid_t dxpl_id, unsigned filters, hsize_t *offset, + size_t data_size, const void *buf); H5_DLL herr_t H5Diterate(void *buf, hid_t type_id, hid_t space_id, H5D_operator_t op, void *operator_data); H5_DLL herr_t H5Dvlen_reclaim(hid_t type_id, hid_t space_id, hid_t plist_id, void *buf); diff --git a/src/H5F.c b/src/H5F.c index 5d13ea6..b1f052d 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -1251,7 +1251,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, * way for us to detect it here anyway). */ if(drvr->cmp) - tent_flags = flags & ~(H5F_ACC_CREAT|H5F_ACC_TRUNC|H5F_ACC_EXCL); + tent_flags = flags & ~(H5F_ACC_CREAT|H5F_ACC_TRUNC|H5F_ACC_EXCL|H5F_ACC_SYNC); else tent_flags = flags; @@ -1454,9 +1454,9 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(!filename || !*filename) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file name") /* In this routine, we only accept the following flags: - * H5F_ACC_EXCL, H5F_ACC_TRUNC and H5F_ACC_DEBUG + * H5F_ACC_SYNC, H5F_ACC_EXCL, H5F_ACC_TRUNC and H5F_ACC_DEBUG */ - if(flags & ~(H5F_ACC_EXCL | H5F_ACC_TRUNC | H5F_ACC_DEBUG)) + if(flags & ~(H5F_ACC_SYNC | H5F_ACC_EXCL | H5F_ACC_TRUNC | H5F_ACC_DEBUG)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags") /* The H5F_ACC_EXCL and H5F_ACC_TRUNC flags are mutually exclusive */ if((flags & H5F_ACC_EXCL) && (flags & H5F_ACC_TRUNC)) diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 4201e07..085465c 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -357,6 +357,8 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) o_flags |= O_CREAT; if(H5F_ACC_EXCL & flags) o_flags |= O_EXCL; + if(H5F_ACC_SYNC & flags) + o_flags |= O_SYNC; /* Open the file */ if((fd = HDopen(name, o_flags, 0666)) < 0) { diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 334879c..2d4678b 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -63,7 +63,8 @@ #define H5F_SUPER_ALL_FLAGS (H5F_SUPER_WRITE_ACCESS | H5F_SUPER_FILE_OK) /* Mask for removing private file access flags */ -#define H5F_ACC_PUBLIC_FLAGS 0x001fu +/* #define H5F_ACC_PUBLIC_FLAGS 0x001fu */ +#define H5F_ACC_PUBLIC_FLAGS 0x003fu /* Free space section+aggregator merge flags */ #define H5F_FS_MERGE_METADATA 0x01 /* Section can merge with metadata aggregator */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 3b3af49..1ef40b0 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -234,7 +234,6 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_FILE_ID(F) ((F)->file_id) #define H5F_PARENT(F) ((F)->parent) #define H5F_NMOUNTS(F) ((F)->nmounts) -#define H5F_DRIVER(F) ((F)->shared->lf) #define H5F_DRIVER_ID(F) ((F)->shared->lf->driver_id) #define H5F_GET_FILENO(F,FILENUM) ((FILENUM) = (F)->shared->lf->fileno) #define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL)) @@ -277,7 +276,6 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_FILE_ID(F) (H5F_get_file_id(F)) #define H5F_PARENT(F) (H5F_get_parent(F)) #define H5F_NMOUNTS(F) (H5F_get_nmounts(F)) -#define H5F_DRIVER(F) (H5F_get_driver(F)) #define H5F_DRIVER_ID(F) (H5F_get_driver_id(F)) #define H5F_GET_FILENO(F,FILENUM) (H5F_get_fileno((F), &(FILENUM))) #define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL)) diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index f32b3e0..7b7acb4 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -48,6 +48,7 @@ #define H5F_ACC_EXCL (H5CHECK 0x0004u) /*fail if file already exists*/ #define H5F_ACC_DEBUG (H5CHECK 0x0008u) /*print debug info */ #define H5F_ACC_CREAT (H5CHECK 0x0010u) /*create non-existing files */ +#define H5F_ACC_SYNC (H5CHECK 0x0020u) /*no filesystem caching */ /* Value passed to H5Pset_elink_acc_flags to cause flags to be taken from the * parent file. */ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 1be5c50..c04ba24 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -919,35 +919,6 @@ H5F_get_driver_id(const H5F_t *f) /*------------------------------------------------------------------------- - * Function: H5F_get_driver - * - * Purpose: Quick and dirty routine to retrieve the file's 'driver' structure - * (Mainly added to stop non-file routines from poking about in the - * H5F_t data structure) - * - * Return: 'driver' structure on success/abort on failure (shouldn't fail) - * - * Programmer: Raymond Lu - * 30 July 2012 - * - *------------------------------------------------------------------------- - */ -H5FD_t * -H5F_get_driver(const H5F_t *f) -{ - /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ - FUNC_ENTER_NOAPI_NOINIT_NOERR - - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); - - FUNC_LEAVE_NOAPI(f->shared->lf) -} /* end H5F_get_driver() */ - - - -/*------------------------------------------------------------------------- * Function: H5F_get_fileno * * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value diff --git a/test/Makefile.am b/test/Makefile.am index 0197e99..006e24b 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -40,7 +40,7 @@ TEST_PROG= testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ pool accum hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ - set_extent ttsafe dectris_tst \ + set_extent ttsafe dectris_tst dectris_perf \ getname vfd ntypes dangle dtransform reserved cross_read \ freespace mf farray earray btree2 fheap file_image @@ -128,7 +128,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse earray.h5 efc[0-5].h5 log_vfd_out.log \ new_multi_file_v16-r.h5 new_multi_file_v16-s.h5 \ split_get_file_image_test-m.h5 split_get_file_image_test-r.h5 \ - file_image_core_test.h5.copy dectris.h5 + file_image_core_test.h5.copy dectris.h5 dectris_perf.h5 # Sources for testhdf5 executable testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ diff --git a/test/Makefile.in b/test/Makefile.in index ddc481e..21706d7 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -89,8 +89,8 @@ am__EXEEXT_1 = testhdf5$(EXEEXT) lheap$(EXEEXT) ohdr$(EXEEXT) \ unlink$(EXEEXT) big$(EXEEXT) mtime$(EXEEXT) fillval$(EXEEXT) \ mount$(EXEEXT) flush1$(EXEEXT) flush2$(EXEEXT) \ app_ref$(EXEEXT) enum$(EXEEXT) set_extent$(EXEEXT) \ - ttsafe$(EXEEXT) dectris_tst$(EXEEXT) getname$(EXEEXT) \ - vfd$(EXEEXT) ntypes$(EXEEXT) dangle$(EXEEXT) \ + ttsafe$(EXEEXT) dectris_tst$(EXEEXT) dectris_perf$(EXEEXT) \ + getname$(EXEEXT) vfd$(EXEEXT) ntypes$(EXEEXT) dangle$(EXEEXT) \ dtransform$(EXEEXT) reserved$(EXEEXT) cross_read$(EXEEXT) \ freespace$(EXEEXT) mf$(EXEEXT) farray$(EXEEXT) earray$(EXEEXT) \ btree2$(EXEEXT) fheap$(EXEEXT) file_image$(EXEEXT) @@ -148,6 +148,10 @@ dangle_SOURCES = dangle.c dangle_OBJECTS = dangle.$(OBJEXT) dangle_LDADD = $(LDADD) dangle_DEPENDENCIES = libh5test.la $(LIBHDF5) +dectris_perf_SOURCES = dectris_perf.c +dectris_perf_OBJECTS = dectris_perf.$(OBJEXT) +dectris_perf_LDADD = $(LDADD) +dectris_perf_DEPENDENCIES = libh5test.la $(LIBHDF5) dectris_tst_SOURCES = dectris_tst.c dectris_tst_OBJECTS = dectris_tst.$(OBJEXT) dectris_tst_LDADD = $(LDADD) @@ -430,23 +434,24 @@ am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) am__v_GEN_0 = @echo " GEN " $@; SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c bittests.c \ btree2.c cache.c cache_api.c cache_tagging.c cmpd_dset.c \ - cross_read.c dangle.c dectris_tst.c dsets.c dt_arith.c \ - dtransform.c dtypes.c earray.c efc.c enum.c err_compat.c \ - error_test.c extend.c external.c farray.c fheap.c file_image.c \ - fillval.c filter_fail.c flush1.c flush2.c freespace.c \ - gen_bad_ohdr.c gen_bogus.c gen_cross.c gen_deflate.c \ - gen_file_image.c gen_filespace.c gen_filters.c gen_new_array.c \ - gen_new_fill.c gen_new_group.c gen_new_mtime.c gen_new_super.c \ - gen_noencoder.c gen_nullspace.c gen_sizes_lheap.c \ - gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \ - istore.c lheap.c links.c links_env.c mf.c mount.c mtime.c \ - ntypes.c objcopy.c ohdr.c pool.c reserved.c set_extent.c \ - space_overflow.c stab.c tcheck_version.c $(testhdf5_SOURCES) \ - testmeta.c $(ttsafe_SOURCES) unlink.c vfd.c + cross_read.c dangle.c dectris_perf.c dectris_tst.c dsets.c \ + dt_arith.c dtransform.c dtypes.c earray.c efc.c enum.c \ + err_compat.c error_test.c extend.c external.c farray.c fheap.c \ + file_image.c fillval.c filter_fail.c flush1.c flush2.c \ + freespace.c gen_bad_ohdr.c gen_bogus.c gen_cross.c \ + gen_deflate.c gen_file_image.c gen_filespace.c gen_filters.c \ + gen_new_array.c gen_new_fill.c gen_new_group.c gen_new_mtime.c \ + gen_new_super.c gen_noencoder.c gen_nullspace.c \ + gen_sizes_lheap.c gen_specmetaread.c gen_udlinks.c getname.c \ + gheap.c hyperslab.c istore.c lheap.c links.c links_env.c mf.c \ + mount.c mtime.c ntypes.c objcopy.c ohdr.c pool.c reserved.c \ + set_extent.c space_overflow.c stab.c tcheck_version.c \ + $(testhdf5_SOURCES) testmeta.c $(ttsafe_SOURCES) unlink.c \ + vfd.c DIST_SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c \ bittests.c btree2.c cache.c cache_api.c cache_tagging.c \ - cmpd_dset.c cross_read.c dangle.c dectris_tst.c dsets.c \ - dt_arith.c dtransform.c dtypes.c earray.c efc.c enum.c \ + cmpd_dset.c cross_read.c dangle.c dectris_perf.c dectris_tst.c \ + dsets.c dt_arith.c dtransform.c dtypes.c earray.c efc.c enum.c \ err_compat.c error_test.c extend.c external.c farray.c fheap.c \ file_image.c fillval.c filter_fail.c flush1.c flush2.c \ freespace.c gen_bad_ohdr.c gen_bogus.c gen_cross.c \ @@ -776,7 +781,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog accum.h5 cmpd_dset.h5 \ earray.h5 efc[0-5].h5 log_vfd_out.log new_multi_file_v16-r.h5 \ new_multi_file_v16-s.h5 split_get_file_image_test-m.h5 \ split_get_file_image_test-r.h5 file_image_core_test.h5.copy \ - dectris.h5 + dectris.h5 dectris_perf.h5 INCLUDES = -I$(top_srcdir)/src -I$(top_builddir)/src # Test script for error_test and err_compat @@ -795,7 +800,7 @@ TEST_PROG = testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ pool accum hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ - set_extent ttsafe dectris_tst \ + set_extent ttsafe dectris_tst dectris_perf \ getname vfd ntypes dangle dtransform reserved cross_read \ freespace mf farray earray btree2 fheap file_image @@ -958,6 +963,9 @@ cross_read$(EXEEXT): $(cross_read_OBJECTS) $(cross_read_DEPENDENCIES) dangle$(EXEEXT): $(dangle_OBJECTS) $(dangle_DEPENDENCIES) @rm -f dangle$(EXEEXT) $(AM_V_CCLD)$(LINK) $(dangle_OBJECTS) $(dangle_LDADD) $(LIBS) +dectris_perf$(EXEEXT): $(dectris_perf_OBJECTS) $(dectris_perf_DEPENDENCIES) + @rm -f dectris_perf$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(dectris_perf_OBJECTS) $(dectris_perf_LDADD) $(LIBS) dectris_tst$(EXEEXT): $(dectris_tst_OBJECTS) $(dectris_tst_DEPENDENCIES) @rm -f dectris_tst$(EXEEXT) $(AM_V_CCLD)$(LINK) $(dectris_tst_OBJECTS) $(dectris_tst_LDADD) $(LIBS) @@ -1160,6 +1168,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cmpd_dset.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cross_read.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dangle.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dectris_perf.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dectris_tst.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsets.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dt_arith.Po@am__quote@ diff --git a/test/dectris_perf.c b/test/dectris_perf.c new file mode 100644 index 0000000..e743a19 --- /dev/null +++ b/test/dectris_perf.c @@ -0,0 +1,535 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * This test is for the DECTRIS project to the H5PSIdirect_write function + * + */ + +#include "h5test.h" +#include +#include +#include +#include +#include +#include +#include + +const char *FILENAME[] = { + "dectris_perf", + "unix.raw", + NULL +}; + +#define DIRECT_DSET "direct_dset" +#define COMPRESSED_DSET "compressed_dset" +#define NO_COMPRESS_DSET "no_compress_dset" +#define RANK 3 +#define NX 100 +#define NY 100 +#define NZ 25 +#define CHUNK_NX 1 +#define CHUNK_NY 100 +#define CHUNK_NZ 25 + +#define DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s))*1.001)+12) +char filename[1024]; +unsigned int *outbuf[NX]; +size_t data_size[NX]; +double total_size = 0.0; +unsigned int *direct_buf[NX]; +double MB = 1048576.0; + +/*-------------------------------------------------- + * Function to report IO rate + *-------------------------------------------------- + */ +void reportTime(struct timeval start, double mbytes) +{ + struct timeval timeval_stop,timeval_diff; + + /*end timing*/ + gettimeofday(&timeval_stop,NULL); + + /* Calculate the elapsed gettimeofday time */ + timeval_diff.tv_usec=timeval_stop.tv_usec-start.tv_usec; + timeval_diff.tv_sec=timeval_stop.tv_sec-start.tv_sec; + + if(timeval_diff.tv_usec<0) { + timeval_diff.tv_usec+=1000000; + timeval_diff.tv_sec--; + } /* end if */ + +/*printf("mbytes=%lf, sec=%lf, usec=%lf\n", mbytes, (double)timeval_diff.tv_sec, (double)timeval_diff.tv_usec);*/ + printf("MBytes/second: %lf\n", (double)mbytes/((double)timeval_diff.tv_sec+((double)timeval_diff.tv_usec/(double)1000000.0))); +} + +/*-------------------------------------------------- + * Create file, datasets, and initialize data + *-------------------------------------------------- + */ +int create_file(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t fapl; + hid_t cparms; + hid_t dataspace, dataset; + hsize_t dims[RANK] = {NX, NY, NZ}; + hsize_t chunk_dims[RANK] ={CHUNK_NX, CHUNK_NY, CHUNK_NZ}; + unsigned int aggression = 9; /* Compression aggression setting */ + int ret; + int i, j, n; + + unsigned int *p; + size_t buf_size = CHUNK_NY*CHUNK_NZ*sizeof(unsigned int); + + const Bytef *z_src; + Bytef *z_dst; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + uLong z_src_nbytes = (uLong)buf_size; + + TESTING("Create a file and dataset"); + + /* + * Create the data space with unlimited dimensions. + */ + if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR; + + /* + * Create a new file. If file exists its contents will be overwritten. + */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + TEST_ERROR; + + /* + * Modify dataset creation properties, i.e. enable chunking and compression + */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + if(H5Pset_chunk( cparms, RANK, chunk_dims) < 0) + TEST_ERROR; + + /* + * Create a new dataset within the file using cparms + * creation properties. + */ + if((dataset = H5Dcreate2(file, NO_COMPRESS_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + /* Set compression */ + if(H5Pset_deflate( cparms, aggression) < 0) + TEST_ERROR; + + if((dataset = H5Dcreate2(file, DIRECT_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + + if((dataset = H5Dcreate2(file, COMPRESSED_DSET, H5T_NATIVE_INT, dataspace, H5P_DEFAULT, + cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if(H5Dclose(dataset) < 0) + TEST_ERROR; + + if(H5Fclose(file) < 0) + TEST_ERROR; + + if(H5Sclose(dataspace) < 0) + TEST_ERROR; + + if(H5Pclose(cparms) < 0) + TEST_ERROR; + + /* Initialize data for chunks */ + for(i = 0; i < NX; i++) { + p = direct_buf[i] = (unsigned int*)malloc(CHUNK_NY*CHUNK_NZ*sizeof(unsigned int)); + + for(j=0; j < CHUNK_NY*CHUNK_NZ; j++, p++) + *p = rand() % 65000; + + z_src = (const Bytef*)direct_buf[i]; + + z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(buf_size); + /* Allocate output (compressed) buffer */ + outbuf[i] = (unsigned int*)malloc((size_t)z_dst_nbytes); + z_dst = (Bytef *)outbuf[i]; + + /* Perform compression from the source to the destination buffer */ + ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + data_size[i] = (size_t)z_dst_nbytes; + total_size += data_size[i]; + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == ret) { + fprintf(stderr, "overflow"); + TEST_ERROR; + } else if(Z_MEM_ERROR == ret) { + fprintf(stderr, "deflate memory error"); + TEST_ERROR; + } else if(Z_OK != ret) { + fprintf(stderr, "other deflate error"); + TEST_ERROR; + } + } + + + PASSED(); + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Sclose(dataspace); + H5Pclose(cparms); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} + +/*-------------------------------------------------- + * Benchmark the performance of the new function + *-------------------------------------------------- + */ +int +test_direct_write(hid_t fapl_id) +{ + hid_t file; /* handles */ + hid_t dataspace, dataset; + hid_t dxpl; + herr_t status; + int i; + + unsigned filter_mask = 0; + hsize_t offset[RANK+1] = {0, 0, 0, 0}; + + struct timeval timeval_start; + + TESTING("H5PSIdirect_write for DECTRIS project"); + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + /* Start the timer */ + gettimeofday(&timeval_start,NULL); + + /* Reopen the file and dataset */ + if((file = H5Fopen(filename, H5F_ACC_RDWR|H5F_ACC_SYNC, fapl_id)) < 0) + TEST_ERROR; + + if((dataset = H5Dopen(file, DIRECT_DSET, H5P_DEFAULT)) < 0) + TEST_ERROR; + + + /* Write the compressed chunk data repeatedly to cover all the chunks in the + * dataset, using the direct writing function. */ + for(i=0; i