diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | src/H5D.c | 226 | ||||
| -rw-r--r-- | src/H5Dcompact.c | 121 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 3 | ||||
| -rw-r--r-- | src/H5Dseq.c | 25 | ||||
| -rw-r--r-- | src/H5F.c | 13 | ||||
| -rw-r--r-- | src/H5Farray.c | 10 | ||||
| -rw-r--r-- | src/H5Fcompact.c | 121 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 9 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 9 | ||||
| -rw-r--r-- | src/H5Fseq.c | 25 | ||||
| -rw-r--r-- | src/H5O.c | 4 | ||||
| -rw-r--r-- | src/H5Olayout.c | 99 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 12 | ||||
| -rw-r--r-- | src/H5Pfapl.c | 6 | ||||
| -rw-r--r-- | src/Makefile.in | 18 | ||||
| -rw-r--r-- | test/dsets.c | 135 | ||||
| -rw-r--r-- | test/fillval.c | 411 | ||||
| -rw-r--r-- | test/tmisc.c | 47 | ||||
| -rw-r--r-- | testpar/t_mdset.c | 111 | ||||
| -rw-r--r-- | testpar/testphdf5.c | 21 |
21 files changed, 1128 insertions, 299 deletions
@@ -791,6 +791,7 @@ ./src/H5F.c ./src/H5Farray.c ./src/H5Fcontig.c +./src/H5Fcompact.c ./src/H5Fistore.c ./src/H5Fpkg.h ./src/H5Fprivate.h @@ -50,10 +50,10 @@ * A dataset is the following struct. */ struct H5D_t { - H5G_entry_t ent; /*cached object header stuff */ - H5T_t *type; /*datatype of this dataset */ - hid_t dcpl_id; /*dataset creation property id */ - H5O_layout_t layout; /*data layout */ + H5G_entry_t ent; /* cached object header stuff */ + H5T_t *type; /* datatype of this dataset */ + hid_t dcpl_id; /* dataset creation property id */ + H5O_layout_t layout; /* data layout */ }; @@ -1500,6 +1500,7 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, H5D_t *new_dset = NULL; H5D_t *ret_value = NULL; int i, ndims; + hsize_t comp_data_size; unsigned u; hsize_t max_dim[H5O_LAYOUT_NDIMS]={0}; H5O_efl_t efl; @@ -1532,8 +1533,12 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve pipeline filter"); if(H5P_get(plist, H5D_CRT_LAYOUT_NAME, &dcpl_layout) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve layout"); + if(H5P_get(plist, H5D_CRT_SPACE_TIME_NAME, &space_time) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve space allocation time"); if(dcpl_pline.nfilters > 0 && H5D_CHUNKED != dcpl_layout) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "filters can only be used with chunked layout"); + if(dcpl_layout==H5D_COMPACT && space_time==H5D_SPACE_ALLOC_LATE) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "compact dataset doesn't support late space allocation"); /* What file is the dataset being added to? */ if (NULL==(f=H5G_insertion_file(loc, name))) @@ -1584,7 +1589,7 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, assert((unsigned)(new_dset->layout.ndims) <= NELMTS(new_dset->layout.dim)); new_dset->layout.dim[new_dset->layout.ndims-1] = H5T_get_size(new_dset->type); - switch (dcpl_layout) { + switch (new_dset->layout.type) { case H5D_CONTIGUOUS: /* * The maximum size of the dataset cannot exceed the storage size. @@ -1643,6 +1648,24 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, new_dset->layout.dim[u] = chunk_size[u]; break; + case H5D_COMPACT: + /* + * Compact dataset is stored in dataset object header message of + * layout. + */ + new_dset->layout.size = H5S_get_simple_extent_npoints(space) * + H5T_get_size(type); + /* Verify data size is smaller than maximum header message size + * (64KB) minus other layout message fields. + */ + comp_data_size=H5O_MAX_SIZE-H5O_layout_meta_size(f, &(new_dset->layout)); + if(new_dset->layout.size > comp_data_size) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "compact dataset size is bigger than header message maximum size"); + if ((ndims=H5S_get_simple_extent_dims(space, new_dset->layout.dim, max_dim))<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize dimension size of compact dataset storage"); + /* remember to check if size is small enough to fit header message */ + break; + default: HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, NULL, "not implemented yet"); } /* end switch */ @@ -1718,11 +1741,13 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to name dataset"); /* - * Initialize storage. We assume that external storage is already - * initialized by the caller, or at least will be before I/O is performed. + * Allocate storage. We assume that external storage is already + * allocated by the caller, or at least will be before I/O is performed. * For parallelization, space is always allocated now except using * external storage. For contiguous layout, space is allocated now if * space allocate time is early; otherwise delay allocation until H5Dwrite. + * For compact dataset, space is allocated regardless of space allocation + * time. */ #ifdef H5_HAVE_PARALLEL if (0==efl.nused) { @@ -1733,7 +1758,8 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, new_dset->layout.addr = HADDR_UNDEF; #else /*H5_HAVE_PARALLEL*/ if (0==efl.nused) { - if(dcpl_layout==H5D_CHUNKED || (dcpl_layout==H5D_CONTIGUOUS && space_time==H5D_SPACE_ALLOC_EARLY)) { + if(dcpl_layout==H5D_CHUNKED || dcpl_layout==H5D_COMPACT || + (dcpl_layout==H5D_CONTIGUOUS && space_time==H5D_SPACE_ALLOC_EARLY)) { if (H5F_arr_create(f, &(new_dset->layout))<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize storage"); } /* end if */ @@ -1744,10 +1770,6 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, new_dset->layout.addr = HADDR_UNDEF; #endif /*H5_HAVE_PARALLEL*/ - /* Update layout message */ - if (H5O_modify (&(new_dset->ent), H5O_LAYOUT, 0, 0, &(new_dset->layout))<0) - HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL, "unable to update layout message"); - /* Update external storage message */ if (efl.nused>0) { size_t heap_size = H5HL_ALIGN (1); @@ -1774,15 +1796,29 @@ H5D_create(H5G_entry_t *loc, const char *name, const H5T_t *type, * 2. layout is contiguous, space allocation time is early, fill value * writing time is upon allocation * 3. external space is treated the same as internal except it's always - * assumed as early for space allocation time. */ + * assumed as early for space allocation time + * 4. compact dataset and fill value writing time is upon allocation + */ #ifdef H5_HAVE_PARALLEL if (H5D_init_storage(new_dset, space)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize storage"); #else /*H5_HAVE_PARALLEL*/ - if(fill_time==H5D_FILL_TIME_ALLOC && ((space_time==H5D_SPACE_ALLOC_EARLY && !efl.nused) || (efl.nused))) - if (H5D_init_storage(new_dset, space)<0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize storage"); + if(fill_time==H5D_FILL_TIME_ALLOC) { + if((dcpl_layout==H5D_CONTIGUOUS && space_time==H5D_SPACE_ALLOC_EARLY) + || (dcpl_layout==H5D_CHUNKED && space_time==H5D_SPACE_ALLOC_EARLY) + || dcpl_layout==H5D_COMPACT ) { + if (H5D_init_storage(new_dset, space)<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize storage"); + } + } + } #endif /*H5_HAVE_PARALLEL*/ + + /* Update layout message */ + if (H5D_COMPACT != new_dset->layout.type && H5O_modify (&(new_dset->ent), H5O_LAYOUT, 0, 0, &(new_dset->layout))<0) + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL, "unable to update layout"); + if (H5D_COMPACT == new_dset->layout.type) + new_dset->layout.dirty = TRUE; /* Success */ ret_value = new_dset; @@ -2048,6 +2084,12 @@ H5D_open_oid(H5G_entry_t *ent) if(H5P_set(plist, H5D_CRT_CHUNK_SIZE_NAME, dataset->layout.dim) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set chunk size"); break; + + case H5D_COMPACT: + layout = H5D_COMPACT; + if(H5P_set(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set layout"); + break; default: HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, NULL, "not implemented yet"); } /* end switch */ @@ -2128,6 +2170,13 @@ H5D_close(H5D_t *dataset) free_failed = (H5T_close(dataset->type) < 0 || H5I_dec_ref(dataset->dcpl_id) < 0); + /*Update header message of layout for compact dataset.*/ + if(dataset->layout.type==H5D_COMPACT && dataset->layout.dirty) { + if(H5O_modify(&(dataset->ent), H5O_LAYOUT, 0, 0, &(dataset->layout))<0) + HRETURN_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update layout message"); + dataset->layout.dirty = FALSE; + } + /* Close the dataset object */ H5O_close(&(dataset->ent)); @@ -2307,7 +2356,10 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, /* If space hasn't been allocated and not using external storage, * return fill value to buffer if fill time is upon allocation, or - * do nothing if fill time is never. */ + * do nothing if fill time is never. If the dataset is compact and + * fill time is NEVER, there is no way to tell whether part of data + * has been overwritten. So just proceed in reading. + */ if(nelmts > 0 && efl.nused==0 && dataset->layout.type==H5D_CONTIGUOUS && dataset->layout.addr==HADDR_UNDEF) { @@ -2373,7 +2425,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, H5_timer_begin(&timer); #endif /* Sanity check dataset, then read it */ - assert(dataset->layout.addr!=HADDR_UNDEF || efl.nused>0); + assert(dataset->layout.addr!=HADDR_UNDEF || efl.nused>0 || dataset->layout.type==H5D_COMPACT); status = (sconv->read)(dataset->ent.file, &(dataset->layout), dc_plist, H5T_get_size(dataset->type), file_space, mem_space, dxpl_id, buf/*out*/); @@ -2667,6 +2719,15 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, if (NULL == (dx_plist = H5P_object_verify(dxpl_id,H5P_DATASET_XFER))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list"); + if (!file_space) { + if (NULL==(free_this_space=H5S_read (&(dataset->ent)))) + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read data space from dataset header"); + file_space = free_this_space; + } /* end if */ + if (!mem_space) + mem_space = file_space; + nelmts = (*mem_space->select.get_npoints)(mem_space); + #ifdef H5_HAVE_PARALLEL /* If MPIO or MPIPOSIX is used, no VL datatype support yet. */ /* This is because they use the global heap in the file and we don't */ @@ -2687,15 +2748,6 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, if (0==(H5F_get_intent(dataset->ent.file) & H5F_ACC_RDWR)) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "no write intent on file"); - if (!file_space) { - if (NULL==(free_this_space=H5S_read (&(dataset->ent)))) - HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read data space from dataset header"); - file_space = free_this_space; - } /* end if */ - if (!mem_space) - mem_space = file_space; - nelmts = (*mem_space->select.get_npoints)(mem_space); - #ifdef H5_HAVE_PARALLEL /* Collect Parallel I/O information for possible later use */ if (H5FD_MPIO==H5P_peek_hid_t(dx_plist,H5D_XFER_VFL_ID_NAME)) { @@ -2706,11 +2758,20 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, xfer_mode = dx->xfer_mode; } } /* end if */ + /* Collective access is not permissible without the MPIO or MPIPOSIX driver */ if (doing_mpio && xfer_mode==H5FD_MPIO_COLLECTIVE && !(IS_H5FD_MPIO(dataset->ent.file) || IS_H5FD_MPIPOSIX(dataset->ent.file))) HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "collective access for MPIO driver only"); + /* If dataset is compact, collective access is only allowed when file space + * selection is H5S_ALL */ + if(doing_mpio && xfer_mode==H5FD_MPIO_COLLECTIVE + && dataset->layout.type==H5D_COMPACT) { + if(file_space->select.type != H5S_SEL_ALL) + HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "collective access to compact dataset doesn't support partial access"); + } + /* Set the "parallel I/O possible" flag, for H5S_find() */ if (H5S_mpi_opt_types_g && IS_H5FD_MPIO(dataset->ent.file)) { /* Only collective write should call this since it eventually @@ -3211,18 +3272,27 @@ H5D_init_storage(H5D_t *dset, const H5S_t *space) if(H5P_get(plist, H5D_CRT_FILL_VALUE_NAME, &fill) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get fill value"); if(H5P_get(plist, H5D_CRT_SPACE_TIME_NAME, &space_time) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve space allocation time"); + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve space allocation time"); switch (dset->layout.type) { + case H5D_COMPACT: + /* + * zero set data buf. If fill value is defined, fall through + * the H5D_CONTIGUOUS case and initialize with fill value. + */ + if(!fill.buf) + HDmemset(dset->layout.buf, 0, dset->layout.size); + case H5D_CONTIGUOUS: /* * If the fill value is set then write it to the entire extent - * of the dataset. + * of the dataset. Note: library default(fill.buf is NULL) is + * not handled here. How to do it? */ snpoints = H5S_get_simple_extent_npoints(space); assert(snpoints>=0); H5_ASSIGN_OVERFLOW(npoints,snpoints,hssize_t,size_t); - + if (fill.buf) { /* * Fill the entire current extent with the fill value. We can do @@ -3355,22 +3425,27 @@ H5D_get_storage_size(H5D_t *dset) FUNC_ENTER_NOAPI(H5D_get_storage_size, 0); - if (H5D_CHUNKED==dset->layout.type) { - ret_value = H5F_istore_allocated(dset->ent.file, dset->layout.ndims, - dset->layout.addr); - } else { - /* Sanity Check */ - assert(dset->layout.type==H5D_CONTIGUOUS); - - /* Datasets which are not allocated yet are using no space on disk */ - if(dset->layout.addr == HADDR_UNDEF) - ret_value=0; - else { - for (u=0, ret_value=1; u<dset->layout.ndims; u++) - ret_value *= dset->layout.dim[u]; - } /* end else */ - } /* end else */ - + switch(dset->layout.type) { + case H5D_CHUNKED: + ret_value = H5F_istore_allocated(dset->ent.file, dset->layout.ndims, + dset->layout.addr); + break; + case H5D_CONTIGUOUS: + /* Datasets which are not allocated yet are using no space on disk */ + if(dset->layout.addr == HADDR_UNDEF) + ret_value=0; + else { + for (u=0, ret_value=1; u<dset->layout.ndims; u++) + ret_value *= dset->layout.dim[u]; + } /* end else */ + break; + case H5D_COMPACT: + ret_value = dset->layout.size; + break; + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset type"); + } + done: FUNC_LEAVE(ret_value); } @@ -4000,6 +4075,64 @@ done: /*------------------------------------------------------------------------- + * Function: H5D_flush + * + * Purpose: Flush any compact datasets cached in memory + * + * Return: Success: Non-negative + * Failure: Negative + * + * + * Programmer: Ray Lu + * + * Date: August 14, 2002 + * + * Comments: Private function + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_flush(H5F_t *f) +{ + unsigned num_dsets; /* Number of datasets in file */ + hid_t *id_list=NULL; /* list of dataset IDs */ + H5D_t *dataset=NULL; /* Dataset pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + unsigned j; /* Index variable */ + + FUNC_ENTER_NOAPI(H5D_flush, FAIL); + + /* Check args */ + assert(f); + + /* Update layout message for compact dataset */ + if(H5F_get_obj_count(f, H5F_OBJ_DATASET, &num_dsets)<0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to get dataset number"); + if(num_dsets>0) { + if(NULL==(id_list=H5MM_malloc(num_dsets*sizeof(hid_t)))) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate memory for ID list"); + if(H5F_get_obj_ids(f, H5F_OBJ_DATASET, id_list)<0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to get dataset ID list"); + for(j=0; j<num_dsets; j++) { + if(NULL==(dataset=H5I_object_verify(id_list[j], H5I_DATASET))) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to get dataset object"); + if(dataset->layout.type==H5D_COMPACT && dataset->layout.dirty) + if(H5O_modify(&(dataset->ent), H5O_LAYOUT, 0, 0, &(dataset->layout))<0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update layout message"); + dataset->layout.dirty = FALSE; + } + } + +done: + if(id_list!=NULL) + H5MM_xfree(id_list); + FUNC_LEAVE(ret_value); +} + + +/*------------------------------------------------------------------------- * Function: H5Ddebug * * Purpose: Prints various information about a dataset. This function is @@ -4039,4 +4172,3 @@ H5Ddebug(hid_t dset_id, unsigned UNUSED flags) done: FUNC_LEAVE(ret_value); } - diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c new file mode 100644 index 0000000..3a36edb --- /dev/null +++ b/src/H5Dcompact.c @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2000-2001 NCSA + * All rights reserved. + * + * Programmer: Raymond Lu <slu@ncsa.uiuc.edu> + * August 5, 2002 + * + * Purpose: Compact dataset I/O functions. These routines are similar + * H5F_contig_* and H5F_istore_*. + */ + +#define H5F_PACKAGE /*suppress error about including H5Fpkg */ + +#include "H5private.h" +#include "H5Eprivate.h" +#include "H5Fpkg.h" +#include "H5Oprivate.h" +#include "H5FDprivate.h" /*file driver */ +#include "H5FLprivate.h" /*Free Lists */ + +/* Interface initialization */ +#define PABLO_MASK H5Fcompact_mask +static int interface_initialize_g = 0; +#define INTERFACE_INIT NULL + +/*------------------------------------------------------------------------- + * Function: H5F_compact_readv + * + * Purpose: Reads some data vectors from a dataset into a buffer. + * The data is in compact dataset. The address is relative + * to the beginning address of the dataset. The offsets and + * sequence lengths are in bytes. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * August 5, 2002 + * + * Notes: + * Offsets in the sequences must be monotonically increasing + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_compact_readv(H5F_t *f, const H5O_layout_t *layout, size_t nseq, + size_t size_arr[], hsize_t offset_arr[], + hid_t dxpl_id, void *_buf/*out*/) +{ + unsigned char *buf=(unsigned char *)_buf; + size_t size; + haddr_t offset; + unsigned u; + herr_t ret_value=SUCCEED; + + FUNC_ENTER_NOAPI(H5F_compact_readv, FAIL); + + for(u=0; u<nseq; u++) { + size=size_arr[u]; + offset=offset_arr[u]; + if(size > 0) { + HDmemcpy(buf, (unsigned char*)layout->buf+offset, size); + buf +=size; + } + } + +done: + FUNC_LEAVE(ret_value); +} /* end H5F_compact_readv() */ + +/*------------------------------------------------------------------------- + * Function: H5F_compact_writev + * + * Purpose: Writes some data vectors from a dataset into a buffer. + * The data is in compact dataset. The address is relative + * to the beginning address for the file. The offsets and + * sequence lengths are in bytes. This function only copies + * data into the buffer in the LAYOUT struct and mark it + * as DIRTY. Later in H5D_close, the data is copied into + * header message in memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * August 5, 2002 + * + * Notes: + * Offsets in the sequences must be monotonically increasing + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_compact_writev(H5F_t *f, H5O_layout_t *layout, size_t nseq, + size_t size_arr[], hsize_t offset_arr[], + hid_t dxpl_id, const void *_buf) +{ + unsigned char *buf=(unsigned char *)_buf; + size_t size; + haddr_t offset; + unsigned u; + herr_t ret_value=SUCCEED; + + FUNC_ENTER_NOAPI(H5F_compact_writev, FAIL); + + for(u=0; u<nseq; u++) { + size=size_arr[u]; + offset=offset_arr[u]; + if(size > 0) { + HDmemcpy((unsigned char*)layout->buf+offset, buf, size); + buf += size; + } + } + + layout->dirty = TRUE; + +done: + FUNC_LEAVE(ret_value); +} /* end H5F_compact_writev */ diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index b0b1609..a60c041 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -142,7 +142,7 @@ #define H5D_XFER_HYPER_VECTOR_SIZE_DEF 1024 typedef struct H5D_t H5D_t; - + /* Functions defined in H5D.c */ __DLL__ herr_t H5D_init(void); __DLL__ H5D_t *H5D_create(H5G_entry_t *loc, const char *name, @@ -175,6 +175,7 @@ __DLL__ herr_t H5D_xfer_copy(hid_t new_plist_id, hid_t old_plist_id, void *copy_data); __DLL__ herr_t H5D_xfer_close(hid_t dxpl_id, void *close_data); __DLL__ herr_t H5D_set_extent(H5D_t *dataset, const hsize_t *size); +__DLL__ herr_t H5D_flush(H5F_t *f); #endif diff --git a/src/H5Dseq.c b/src/H5Dseq.c index a3f2941..f3f70de 100644 --- a/src/H5Dseq.c +++ b/src/H5Dseq.c @@ -102,7 +102,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_seq_write(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, +H5F_seq_write(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, H5P_genplist_t *dc_plist, const H5S_t *file_space, size_t elmt_size, size_t seq_len, hsize_t file_offset, const void *buf) @@ -215,7 +215,7 @@ H5F_seq_readv(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, } /* Collective MPIO access is unsupported for non-contiguous datasets */ - if (H5D_CONTIGUOUS!=layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) + if (H5D_CHUNKED==layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) HGOTO_ERROR (H5E_DATASET, H5E_READERROR, FAIL, "collective access on non-contiguous datasets not supported yet"); #endif /* H5_HAVE_PARALLEL */ @@ -498,6 +498,13 @@ H5F_seq_readv(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, break; + case H5D_COMPACT: + + /* Pass along the vector of sequences to read */ + if (H5F_compact_readv(f, layout, nseq, seq_len_arr, file_offset_arr, dxpl_id, real_buf)<0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "block read failed"); + + break; default: assert("not implemented yet" && 0); HGOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, "unsupported storage layout"); @@ -532,7 +539,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_seq_writev(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, +H5F_seq_writev(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, H5P_genplist_t *dc_plist, const H5S_t *file_space, size_t elmt_size, size_t nseq, size_t seq_len_arr[], hsize_t file_offset_arr[], @@ -599,8 +606,8 @@ H5F_seq_writev(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, } /* Collective MPIO access is unsupported for non-contiguous datasets */ - if (H5D_CONTIGUOUS!=layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) - HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "collective access on non-contiguous datasets not supported yet"); + if (H5D_CHUNKED==layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) + HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "collective access on chunked datasets not supported yet"); #endif /* H5_HAVE_PARALLEL */ /* Get necessary properties from property list */ @@ -880,6 +887,14 @@ H5F_seq_writev(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, break; + case H5D_COMPACT: + + /* Pass along the vector of sequences to write */ + if (H5F_compact_writev(f, layout, nseq, seq_len_arr, file_offset_arr, dxpl_id, real_buf)<0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); + + break; + default: assert("not implemented yet" && 0); HGOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, "unsupported storage layout"); @@ -69,9 +69,6 @@ static herr_t H5F_flush(H5F_t *f, H5F_scope_t scope, hbool_t invalidate, hbool_t alloc_only, hbool_t closing); static haddr_t H5F_locate_signature(H5FD_t *file); static int H5F_flush_all_cb(H5F_t *f, hid_t fid, const void *_invalidate); -static herr_t H5F_get_obj_count(H5F_t *f, unsigned types, - unsigned *obj_id_count); -static herr_t H5F_get_obj_ids(H5F_t *f, unsigned types, hid_t *obj_id_list); static herr_t H5F_get_objects(H5F_t *f, unsigned types, hid_t *obj_id_list, unsigned *obj_id_count); static herr_t H5F_get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); @@ -943,7 +940,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5F_get_obj_count(H5F_t *f, unsigned types, unsigned *obj_id_count) { herr_t ret_value = SUCCEED; @@ -1010,7 +1007,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5F_get_obj_ids(H5F_t *f, unsigned types, hid_t *oid_list) { herr_t ret_value = SUCCEED; @@ -2318,7 +2315,7 @@ H5F_flush(H5F_t *f, H5F_scope_t scope, hbool_t invalidate, int btree_k[H5B_NUM_BTREE_ID]; /* B-tree size info */ unsigned sym_leaf_k; /* Number of symbols in B-tree leafs */ H5P_genplist_t *plist; /* Property list */ - herr_t ret_value; /* Return value */ + herr_t ret_value; /* Return value */ FUNC_ENTER_NOINIT(H5F_flush); @@ -2346,6 +2343,10 @@ H5F_flush(H5F_t *f, H5F_scope_t scope, hbool_t invalidate, /* Avoid flushing buffers & caches when alloc_only set */ if(!alloc_only) { + /* flush any cached compact storage raw data */ + if (H5D_flush(f)<0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush dataset cache"); + /* If we are invalidating everything (which only happens just before * the file closes), release the unused portion of the metadata and * "small data" blocks back to the free lists in the file. diff --git a/src/H5Farray.c b/src/H5Farray.c index 1e389f4..e3657d0 100644 --- a/src/H5Farray.c +++ b/src/H5Farray.c @@ -80,6 +80,14 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/) HGOTO_ERROR (H5E_IO, H5E_CANTINIT, FAIL, "unable to initialize chunked storage"); break; + case H5D_COMPACT: + /* Reserve space in layout header message for the entire array. */ + assert(layout->size>0); + if (NULL==(layout->buf=H5MM_malloc(layout->size))) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate memory for compact dataset"); + layout->dirty = TRUE; + break; + default: assert ("not implemented yet" && 0); HGOTO_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL, @@ -366,7 +374,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_arr_write(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, +H5F_arr_write(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, H5P_genplist_t *dc_plist, const hsize_t _hslab_size[], const hsize_t mem_size[], const hssize_t mem_offset[], const hssize_t file_offset[], diff --git a/src/H5Fcompact.c b/src/H5Fcompact.c new file mode 100644 index 0000000..3a36edb --- /dev/null +++ b/src/H5Fcompact.c @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2000-2001 NCSA + * All rights reserved. + * + * Programmer: Raymond Lu <slu@ncsa.uiuc.edu> + * August 5, 2002 + * + * Purpose: Compact dataset I/O functions. These routines are similar + * H5F_contig_* and H5F_istore_*. + */ + +#define H5F_PACKAGE /*suppress error about including H5Fpkg */ + +#include "H5private.h" +#include "H5Eprivate.h" +#include "H5Fpkg.h" +#include "H5Oprivate.h" +#include "H5FDprivate.h" /*file driver */ +#include "H5FLprivate.h" /*Free Lists */ + +/* Interface initialization */ +#define PABLO_MASK H5Fcompact_mask +static int interface_initialize_g = 0; +#define INTERFACE_INIT NULL + +/*------------------------------------------------------------------------- + * Function: H5F_compact_readv + * + * Purpose: Reads some data vectors from a dataset into a buffer. + * The data is in compact dataset. The address is relative + * to the beginning address of the dataset. The offsets and + * sequence lengths are in bytes. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * August 5, 2002 + * + * Notes: + * Offsets in the sequences must be monotonically increasing + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_compact_readv(H5F_t *f, const H5O_layout_t *layout, size_t nseq, + size_t size_arr[], hsize_t offset_arr[], + hid_t dxpl_id, void *_buf/*out*/) +{ + unsigned char *buf=(unsigned char *)_buf; + size_t size; + haddr_t offset; + unsigned u; + herr_t ret_value=SUCCEED; + + FUNC_ENTER_NOAPI(H5F_compact_readv, FAIL); + + for(u=0; u<nseq; u++) { + size=size_arr[u]; + offset=offset_arr[u]; + if(size > 0) { + HDmemcpy(buf, (unsigned char*)layout->buf+offset, size); + buf +=size; + } + } + +done: + FUNC_LEAVE(ret_value); +} /* end H5F_compact_readv() */ + +/*------------------------------------------------------------------------- + * Function: H5F_compact_writev + * + * Purpose: Writes some data vectors from a dataset into a buffer. + * The data is in compact dataset. The address is relative + * to the beginning address for the file. The offsets and + * sequence lengths are in bytes. This function only copies + * data into the buffer in the LAYOUT struct and mark it + * as DIRTY. Later in H5D_close, the data is copied into + * header message in memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * August 5, 2002 + * + * Notes: + * Offsets in the sequences must be monotonically increasing + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_compact_writev(H5F_t *f, H5O_layout_t *layout, size_t nseq, + size_t size_arr[], hsize_t offset_arr[], + hid_t dxpl_id, const void *_buf) +{ + unsigned char *buf=(unsigned char *)_buf; + size_t size; + haddr_t offset; + unsigned u; + herr_t ret_value=SUCCEED; + + FUNC_ENTER_NOAPI(H5F_compact_writev, FAIL); + + for(u=0; u<nseq; u++) { + size=size_arr[u]; + offset=offset_arr[u]; + if(size > 0) { + HDmemcpy((unsigned char*)layout->buf+offset, buf, size); + buf += size; + } + } + + layout->dirty = TRUE; + +done: + FUNC_LEAVE(ret_value); +} /* end H5F_compact_writev */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 189cbc7..92e6a5b 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -210,5 +210,14 @@ __DLL__ herr_t H5F_contig_readv(H5F_t *f, hsize_t max_data, H5FD_mem_t type, had __DLL__ herr_t H5F_contig_writev(H5F_t *f, hsize_t max_data, H5FD_mem_t type, haddr_t addr, size_t nseq, size_t size[], hsize_t offset[], hid_t dxpl_id, const void *buf); +/* Functions that operate on compact dataset storage */ +__DLL__ herr_t H5F_compact_readv(H5F_t *f, const struct H5O_layout_t *layout, size_t nseq, + size_t size_arr[], hsize_t offset_arr[], + hid_t dxpl_id, void *_buf/*out*/); +__DLL__ herr_t H5F_compact_writev(H5F_t *f, struct H5O_layout_t *layout, size_t nseq, + size_t size_arr[], hsize_t offset_arr[], + hid_t dxpl_id, const void *_buf); + + #endif diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index dd7cbd9..15ef548 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -340,6 +340,9 @@ __DLL__ herr_t H5F_init(void); __DLL__ unsigned H5F_get_intent(const H5F_t *f); __DLL__ hid_t H5F_get_driver_id(const H5F_t *f); __DLL__ herr_t H5F_get_fileno(const H5F_t *f, unsigned long *filenum); +__DLL__ herr_t H5F_get_obj_count(H5F_t *f, unsigned types, + unsigned *obj_id_count); +__DLL__ herr_t H5F_get_obj_ids(H5F_t *f, unsigned types, hid_t *obj_id_list); /* Functions that operate on array storage */ __DLL__ herr_t H5F_arr_create(H5F_t *f, @@ -352,7 +355,7 @@ __DLL__ herr_t H5F_arr_read (H5F_t *f, hid_t dxpl_id, const hssize_t mem_offset[], const hssize_t file_offset[], void *_buf/*out*/); __DLL__ herr_t H5F_arr_write (H5F_t *f, hid_t dxpl_id, - const struct H5O_layout_t *layout, + struct H5O_layout_t *layout, struct H5P_genplist_t *dc_plist, const hsize_t _hslab_size[], const hsize_t mem_size[], @@ -372,7 +375,7 @@ __DLL__ herr_t H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5S_t *file_space, size_t elmt_size, size_t seq_len, hsize_t file_offset, void *_buf/*out*/); __DLL__ herr_t H5F_seq_write (H5F_t *f, hid_t dxpl_id, - const struct H5O_layout_t *layout, + struct H5O_layout_t *layout, struct H5P_genplist_t *dc_plist, const struct H5S_t *file_space, size_t elmt_size, size_t seq_len, hsize_t file_offset, const void *_buf); @@ -384,7 +387,7 @@ __DLL__ herr_t H5F_seq_readv(H5F_t *f, hid_t dxpl_id, const struct H5S_t *file_space, size_t elmt_size, size_t nseq, size_t seq_len[], hsize_t file_offset[], void *_buf/*out*/); __DLL__ herr_t H5F_seq_writev(H5F_t *f, hid_t dxpl_id, - const struct H5O_layout_t *layout, + struct H5O_layout_t *layout, struct H5P_genplist_t *dc_plist, const struct H5S_t *file_space, size_t elmt_size, size_t nseq, size_t seq_len[], hsize_t file_offset[], const void *_buf); diff --git a/src/H5Fseq.c b/src/H5Fseq.c index a3f2941..f3f70de 100644 --- a/src/H5Fseq.c +++ b/src/H5Fseq.c @@ -102,7 +102,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_seq_write(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, +H5F_seq_write(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, H5P_genplist_t *dc_plist, const H5S_t *file_space, size_t elmt_size, size_t seq_len, hsize_t file_offset, const void *buf) @@ -215,7 +215,7 @@ H5F_seq_readv(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, } /* Collective MPIO access is unsupported for non-contiguous datasets */ - if (H5D_CONTIGUOUS!=layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) + if (H5D_CHUNKED==layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) HGOTO_ERROR (H5E_DATASET, H5E_READERROR, FAIL, "collective access on non-contiguous datasets not supported yet"); #endif /* H5_HAVE_PARALLEL */ @@ -498,6 +498,13 @@ H5F_seq_readv(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, break; + case H5D_COMPACT: + + /* Pass along the vector of sequences to read */ + if (H5F_compact_readv(f, layout, nseq, seq_len_arr, file_offset_arr, dxpl_id, real_buf)<0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "block read failed"); + + break; default: assert("not implemented yet" && 0); HGOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, "unsupported storage layout"); @@ -532,7 +539,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_seq_writev(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, +H5F_seq_writev(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, H5P_genplist_t *dc_plist, const H5S_t *file_space, size_t elmt_size, size_t nseq, size_t seq_len_arr[], hsize_t file_offset_arr[], @@ -599,8 +606,8 @@ H5F_seq_writev(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, } /* Collective MPIO access is unsupported for non-contiguous datasets */ - if (H5D_CONTIGUOUS!=layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) - HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "collective access on non-contiguous datasets not supported yet"); + if (H5D_CHUNKED==layout->type && H5FD_MPIO_COLLECTIVE==xfer_mode) + HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "collective access on chunked datasets not supported yet"); #endif /* H5_HAVE_PARALLEL */ /* Get necessary properties from property list */ @@ -880,6 +887,14 @@ H5F_seq_writev(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, break; + case H5D_COMPACT: + + /* Pass along the vector of sequences to write */ + if (H5F_compact_writev(f, layout, nseq, seq_len_arr, file_offset_arr, dxpl_id, real_buf)<0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); + + break; + default: assert("not implemented yet" && 0); HGOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, "unsupported storage layout"); @@ -584,7 +584,7 @@ H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5O_t *oh) id = oh->mesg[u].type->id; UINT16ENCODE(p, id); - assert (oh->mesg[u].raw_size<65536); + assert (oh->mesg[u].raw_size<H5O_MAX_SIZE); UINT16ENCODE(p, oh->mesg[u].raw_size); *p++ = oh->mesg[u].flags; *p++ = 0; /*reserved*/ @@ -1280,7 +1280,7 @@ H5O_modify(H5G_entry_t *ent, const H5O_class_t *type, int overwrite, } if (0==(flags & H5O_FLAG_SHARED)) { size = (type->raw_size) (ent->file, mesg); - if (size>=65536) + if (size>=H5O_MAX_SIZE) HGOTO_ERROR (H5E_OHDR, H5E_CANTINIT, FAIL, "object header message is too large (16k max)"); } idx = H5O_alloc(ent->file, oh, type, size); diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 7a65fb9..65f4cdb 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -81,7 +81,7 @@ H5O_layout_decode(H5F_t *f, const uint8_t *p, H5O_shared_t UNUSED *sh) { H5O_layout_t *mesg = NULL; int version; - unsigned u; + unsigned u; void *ret_value; /* Return value */ FUNC_ENTER_NOAPI(H5O_layout_decode, NULL); @@ -107,18 +107,31 @@ H5O_layout_decode(H5F_t *f, const uint8_t *p, H5O_shared_t UNUSED *sh) /* Layout class */ mesg->type = *p++; - assert(H5D_CONTIGUOUS == mesg->type || H5D_CHUNKED == mesg->type); + assert(H5D_CONTIGUOUS == mesg->type || H5D_CHUNKED == mesg->type || H5D_COMPACT == mesg->type); /* Reserved bytes */ p += 5; /* Address */ - H5F_addr_decode(f, &p, &(mesg->addr)); + if(mesg->type!=H5D_COMPACT) + H5F_addr_decode(f, &p, &(mesg->addr)); /* Read the size */ for (u = 0; u < mesg->ndims; u++) UINT32DECODE(p, mesg->dim[u]); + if(mesg->type == H5D_COMPACT) { + UINT32DECODE(p, mesg->size); + if(mesg->size > 0) { + if(NULL==(mesg->buf=H5MM_malloc(mesg->size))) { + HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed for fill value"); + } + HDmemcpy(mesg->buf, p, mesg->size); + p += mesg->size; + } + } + /* Set return value */ ret_value=mesg; @@ -155,7 +168,7 @@ static herr_t H5O_layout_encode(H5F_t *f, uint8_t *p, const void *_mesg) { const H5O_layout_t *mesg = (const H5O_layout_t *) _mesg; - unsigned u; + unsigned u; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5O_layout_encode, FAIL); @@ -172,7 +185,9 @@ H5O_layout_encode(H5F_t *f, uint8_t *p, const void *_mesg) *p++ = H5O_LAYOUT_VERSION_2; else *p++ = H5O_LAYOUT_VERSION_1; - } else + } else if(mesg->type==H5D_COMPACT) { + *p++ = H5O_LAYOUT_VERSION_2; + } else *p++ = H5O_LAYOUT_VERSION_1; /* number of dimensions */ @@ -186,12 +201,22 @@ H5O_layout_encode(H5F_t *f, uint8_t *p, const void *_mesg) *p++ = 0; /* data or B-tree address */ - H5F_addr_encode(f, &p, mesg->addr); + if(mesg->type!=H5D_COMPACT) + H5F_addr_encode(f, &p, mesg->addr); /* dimension size */ for (u = 0; u < mesg->ndims; u++) UINT32ENCODE(p, mesg->dim[u]); + if(mesg->type==H5D_COMPACT) { + UINT32ENCODE(p, mesg->size); + if(mesg->size>0 && mesg->buf) { + H5_CHECK_OVERFLOW(mesg->size,ssize_t,size_t); + HDmemcpy(p, mesg->buf, mesg->size); + p += mesg->size; + } + } + done: FUNC_LEAVE(ret_value); } @@ -240,11 +265,58 @@ done: /*------------------------------------------------------------------------- + * Function: H5O_layout_meta_size + * + * Purpose: Returns the size of the raw message in bytes except raw data + * part for compact dataset. This function doesn't take into + * account message alignment. + * + * Return: Success: Message data size in bytes(except raw data + * for compact dataset) + * Failure: 0 + * + * Programmer: Raymond Lu + * August 14, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5O_layout_meta_size(H5F_t *f, const void *_mesg) +{ + const H5O_layout_t *mesg = (const H5O_layout_t *) _mesg; + size_t ret_value; + + FUNC_ENTER_NOAPI(H5O_layout_meta_size, 0); + + /* check args */ + assert(f); + assert(mesg); + assert(mesg->ndims > 0 && mesg->ndims <= H5O_LAYOUT_NDIMS); + + ret_value = 1 + /* Version number */ + 1 + /* layout class type */ + 1 + /* dimensionality */ + 5 + /* reserved bytes */ + mesg->ndims * 4; /* size of each dimension */ + + if(mesg->type==H5D_COMPACT) + ret_value += 4; /* size field for compact dataset */ + else + ret_value += H5F_SIZEOF_ADDR(f); /* file address of data or B-tree for chunked dataset */ + +done: + FUNC_LEAVE(ret_value); +} + + +/*------------------------------------------------------------------------- * Function: H5O_layout_size * - * Purpose: Returns the size of the raw message in bytes not counting the - * message type or size fields, but only the data fields. This - * function doesn't take into account message alignment. + * Purpose: Returns the size of the raw message in bytes. If it's + * compact dataset, the data part is also included. + * This function doesn't take into account message alignment. * * Return: Success: Message data size in bytes * @@ -270,12 +342,9 @@ H5O_layout_size(H5F_t *f, const void *_mesg) assert(mesg); assert(mesg->ndims > 0 && mesg->ndims <= H5O_LAYOUT_NDIMS); - ret_value = H5F_SIZEOF_ADDR(f) + /* B-tree address */ - 1 + /* max dimension index */ - 1 + /* layout class number */ - 6 + /* reserved bytes */ - mesg->ndims * 4; /* alignment */ - + ret_value = H5O_layout_meta_size(f, mesg); + if(mesg->type==H5D_COMPACT) + ret_value += mesg->size;/* data for compact dataset */ done: FUNC_LEAVE(ret_value); } diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 786ae69..8c66bcc 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -38,6 +38,7 @@ #define H5O_ALIGN(X) (8*(((X)+8-1)/8)) #define H5O_MIN_SIZE H5O_ALIGN(32) /*min obj header data size */ +#define H5O_MAX_SIZE 65536 /*max obj header data size */ #define H5O_NMESGS 32 /*initial number of messages */ #define H5O_NCHUNKS 8 /*initial number of chunks */ #define H5O_NEW_MESG (-1) /*new message */ @@ -165,7 +166,6 @@ typedef struct H5O_fill_new_t { htri_t fill_defined; /* whether fill value is defined */ } H5O_fill_new_t; - /* * External File List Message */ @@ -197,9 +197,12 @@ __DLLVAR__ const H5O_class_t H5O_LAYOUT[1]; typedef struct H5O_layout_t { int type; /*type of layout, H5D_layout_t */ - haddr_t addr; /*file address of data or B-tree */ + haddr_t addr; /*file address of data or B-tree */ unsigned ndims; /*num dimensions in stored data */ - hsize_t dim[H5O_LAYOUT_NDIMS]; /*size of data or chunk */ + hsize_t dim[H5O_LAYOUT_NDIMS]; /*size of data or chunk in bytes */ + hbool_t dirty; /*dirty flag for compact dataset */ + size_t size; /*size of compact dataset in bytes */ + void *buf; /*buffer for compact dataset */ } H5O_layout_t; /* @@ -312,6 +315,9 @@ __DLL__ herr_t H5O_share(H5F_t *f, const H5O_class_t *type, const void *mesg, __DLL__ herr_t H5O_debug(H5F_t *f, haddr_t addr, FILE * stream, int indent, int fwidth); +/* Layout operators */ +__DLL__ size_t H5O_layout_meta_size(H5F_t *f, const void *_mesg); + /* EFL operators */ __DLL__ hsize_t H5O_efl_total_size(H5O_efl_t *efl); __DLL__ herr_t H5O_efl_read(H5F_t *f, const H5O_efl_t *efl, haddr_t addr, diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index a20a1b1..c58711e 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -882,12 +882,12 @@ done: /*------------------------------------------------------------------------- * Function: H5Pget_fclose_degree * - * Purpose: Returns the current setting for the garbage collection + * Purpose: Returns the degree for the file close behavior. * * Return: Non-negative on success/Negative on failure * - * Programmer: Quincey Koziol - * June, 1999 + * Programmer: Raymond Lu + * November, 2001 * * Modifications: * diff --git a/src/Makefile.in b/src/Makefile.in index 4f7a155..2b5d416 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -20,15 +20,15 @@ CLEAN=libhdf5.settings ## Source and object files for the library (lexicographically)... LIB_SRC=H5.c H5A.c H5AC.c H5B.c H5D.c H5E.c H5F.c H5Farray.c H5Fcontig.c \ - H5Fistore.c H5Fseq.c H5FD.c H5FDcore.c H5FDfamily.c H5FDgass.c H5FDlog.c \ - H5FDmpio.c H5FDmpiposix.c H5FDmulti.c H5FDsec2.c H5FDsrb.c H5FDstdio.c \ - H5FDstream.c H5FL.c H5G.c H5Gent.c H5Gnode.c H5Gstab.c H5HG.c H5HL.c H5I.c \ - H5MF.c H5MM.c H5O.c H5Oattr.c H5Ocomp.c H5Ocont.c H5Odtype.c H5Oefl.c \ - H5Ofill.c H5Olayout.c H5Omtime.c H5Oname.c H5Onull.c H5Osdspace.c \ - H5Oshared.c H5Ostab.c H5P.c H5Pdcpl.c H5Pdxpl.c H5Pfapl.c H5Pfcpl.c H5R.c \ - H5S.c H5Sall.c H5Shyper.c H5Smpio.c H5Snone.c H5Spoint.c H5Sselect.c H5T.c \ - H5Tbit.c H5Tconv.c H5Tinit.c H5Tvlen.c H5TB.c H5TS.c H5V.c H5Z.c \ - H5Zdeflate.c + H5Fcompact.c H5Fistore.c H5Fseq.c H5FD.c H5FDcore.c H5FDfamily.c \ + H5FDgass.c H5FDlog.c H5FDmpio.c H5FDmpiposix.c H5FDmulti.c H5FDsec2.c \ + H5FDsrb.c H5FDstdio.c H5FDstream.c H5FL.c H5G.c H5Gent.c H5Gnode.c \ + H5Gstab.c H5HG.c H5HL.c H5I.c H5MF.c H5MM.c H5O.c H5Oattr.c H5Ocomp.c \ + H5Ocont.c H5Odtype.c H5Oefl.c H5Ofill.c H5Olayout.c H5Omtime.c H5Oname.c \ + H5Onull.c H5Osdspace.c H5Oshared.c H5Ostab.c H5P.c H5Pdcpl.c H5Pdxpl.c \ + H5Pfapl.c H5Pfcpl.c H5R.c H5S.c H5Sall.c H5Shyper.c H5Smpio.c H5Snone.c \ + H5Spoint.c H5Sselect.c H5T.c H5Tbit.c H5Tconv.c H5Tinit.c H5Tvlen.c \ + H5TB.c H5TS.c H5V.c H5Z.c H5Zdeflate.c LIB_OBJ=$(LIB_SRC:.c=.lo) diff --git a/test/dsets.c b/test/dsets.c index 890aa2f..73f9392 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -12,12 +12,15 @@ const char *FILENAME[] = { "dataset", + "compact_dataset", NULL }; #define DSET_DEFAULT_NAME "default" #define DSET_CHUNKED_NAME "chunked" +#define DSET_COMPACT_NAME "compact" #define DSET_SIMPLE_IO_NAME "simple_io" +#define DSET_COMPACT_IO_NAME "compact_io" #define DSET_TCONV_NAME "tconv" #define DSET_COMPRESS_NAME "compressed" #define DSET_BOGUS_NAME "bogus" @@ -41,14 +44,17 @@ int points[100][200], check[100][200]; * Tuesday, December 9, 1997 * * Modifications: + * Added test for compact dataset creation. + * Raymond Lu + * August 8, 2002 * *------------------------------------------------------------------------- */ static herr_t test_create(hid_t file) { - hid_t dataset, space, create_parms; - hsize_t dims[2]; + hid_t dataset, space, small_space, create_parms; + hsize_t dims[2], small_dims[2]; herr_t status; hsize_t csize[2]; @@ -60,6 +66,12 @@ test_create(hid_t file) space = H5Screate_simple(2, dims, NULL); assert(space>=0); + /* Create a small data space for compact dataset */ + small_dims[0] = 16; + small_dims[1] = 8; + small_space = H5Screate_simple(2, small_dims, NULL); + assert(space>=0); + /* * Create a dataset using the default dataset creation properties. We're * not sure what they are, so we won't check. @@ -148,6 +160,22 @@ test_create(hid_t file) */ if (H5Dclose(dataset) < 0) goto error; + /* + * Create a compact dataset, then close it. + */ + create_parms = H5Pcreate(H5P_DATASET_CREATE); + assert(create_parms >= 0); + status = H5Pset_layout(create_parms, H5D_COMPACT); + assert(status >= 0); + status = H5Pset_space_time(create_parms, H5D_SPACE_ALLOC_EARLY); + assert(status >= 0); + + dataset = H5Dcreate(file, DSET_COMPACT_NAME, H5T_NATIVE_DOUBLE, + small_space, create_parms); + if(dataset < 0) goto error; + H5Pclose(create_parms); + if(H5Dclose(dataset) <0) goto error; + PASSED(); return 0; @@ -235,6 +263,108 @@ test_simple_io(hid_t file) error: return -1; } + + +/*------------------------------------------------------------------------- + * Function: test_compact_io + * + * Purpose: Tests compact dataset I/O. That is, reading and writing a + * complete multi-dimensional array without data type or data + * space conversions, without compression, and store in + * compact dataset. + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Raymond Lu + * August 8, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +test_compact_io(void) +{ + hid_t file, dataset, space, plist; + hsize_t dims[2]; + herr_t status; + int wbuf[16][8], rbuf[16][8]; + int i, j, n; + + TESTING("compact dataset I/O"); + + /* Initialize data */ + n=0; + for(i=0; i<16; i++) { + for(j=0; j<8; j++) { + wbuf[i][j] = n++; + } + } + + /* Create a small data space for compact dataset */ + dims[0] = 16; + dims[1] = 8; + space = H5Screate_simple(2, dims, NULL); + assert(space>=0); + + /* Create a file */ + if((file=H5Fcreate(FILENAME[1], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))<0) + goto error; + + /* Create property list for compact dataset creation */ + plist = H5Pcreate(H5P_DATASET_CREATE); + assert(plist >= 0); + status = H5Pset_layout(plist, H5D_COMPACT); + assert(status >= 0); + status = H5Pset_space_time(plist, H5D_SPACE_ALLOC_EARLY); + assert(status >= 0); + + /* Create and write to a compact dataset */ + if((dataset = H5Dcreate(file, DSET_COMPACT_IO_NAME, H5T_NATIVE_INT, space, + plist))<0) + goto error; + if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf)<0) + goto error; + + /* Close file */ + H5Sclose(space); + H5Pclose(plist); + H5Dclose(dataset); + H5Fclose(file); + + /* + * Open the file and check data + */ + if((file=H5Fopen(FILENAME[1], H5F_ACC_RDONLY, H5P_DEFAULT))<0) + goto error; + if((dataset = H5Dopen(file, DSET_COMPACT_IO_NAME))<0) + goto error; + if(H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf)<0) + goto error; + + /* Check that the values read are the same as the values written */ + for (i = 0; i < 16; i++) { + for (j = 0; j < 8; j++) { + if (rbuf[i][j] != wbuf[i][j]) { + H5_FAILED(); + printf(" Read different values than written.\n"); + printf(" At index %d,%d\n", i, j); + goto error; + } + } + } + + H5Dclose(dataset); + H5Fclose(file); + PASSED(); + return 0; + + error: + return -1; +} + /*------------------------------------------------------------------------- * Function: test_tconv @@ -868,6 +998,7 @@ main(void) nerrors += test_create(file)<0 ?1:0; nerrors += test_simple_io(file)<0 ?1:0; + nerrors += test_compact_io()<0 ?1:0; nerrors += test_tconv(file)<0 ?1:0; nerrors += test_compression(file)<0 ?1:0; nerrors += test_multiopen (file)<0 ?1:0; diff --git a/test/fillval.c b/test/fillval.c index fd9045f..c9925f9 100644 --- a/test/fillval.c +++ b/test/fillval.c @@ -22,6 +22,8 @@ const char *FILENAME[] = { "fillval_4", "fillval_5", "fillval_6", + "fillval_7", + "fillval_8", NULL }; @@ -226,7 +228,7 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) hid_t file=-1, space=-1, dcpl=-1, comp_type_id=-1; hid_t dset1=-1, dset2=-1, dset3=-1, dset4=-1, dset5=-1, dset6=-1, /* dset7=-1, */ dset8=-1, dset9=-1; - hsize_t cur_size[5] = {32, 16, 8, 4, 2}; + hsize_t cur_size[5] = {2, 16, 8, 4, 2}; hsize_t ch_size[5] = {1, 1, 1, 4, 2}; short rd_s, fill_s = 0x1234; long rd_l, fill_l = 0x4321; @@ -238,6 +240,8 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) if (H5D_CHUNKED==layout) { TESTING("chunked dataset creation"); + } else if (H5D_COMPACT==layout) { + TESTING("compact dataset creation"); } else { TESTING("contiguous dataset creation"); } @@ -252,68 +256,74 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) if ((dcpl=H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5D_CHUNKED==layout) { if (H5Pset_chunk(dcpl, 5, ch_size)<0) goto error; + } else if (H5D_COMPACT==layout) { + if (H5Pset_layout(dcpl, H5D_COMPACT)<0) goto error; } + /* Create a compound datatype */ if((comp_type_id = create_compound_type())<0) goto error; - /* I. Test cases for late space allocation */ - if(H5Pset_space_time(dcpl, H5D_SPACE_ALLOC_LATE) < 0) goto error; - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; + /* I. Test cases for late space allocation except compact dataset */ - /* 1. Compound datatype test */ - if(H5Pget_fill_value(dcpl, comp_type_id, &fill_ctype)<0) goto error; - fill_ctype.y = 4444; - if(H5Pset_fill_value(dcpl, comp_type_id, &fill_ctype)<0) goto error; - if((dset9 = H5Dcreate(file, "dset9", comp_type_id, space, dcpl))<0) - goto error; + if(H5D_COMPACT!=layout) { + if(H5Pset_space_time(dcpl, H5D_SPACE_ALLOC_LATE) < 0) goto error; + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; - /* The three datasets test three fill - * conversion paths: small to large, large to small, and no conversion. - * They depend on `short' being smaller than `long'. - */ - /* 2. Small to large fill conversion */ + /* 1. Compound datatype test */ + if(H5Pget_fill_value(dcpl, comp_type_id, &fill_ctype)<0) goto error; + fill_ctype.y = 4444; + if(H5Pset_fill_value(dcpl, comp_type_id, &fill_ctype)<0) goto error; + if((dset9 = H5Dcreate(file, "dset9", comp_type_id, space, dcpl))<0) + goto error; + + /* The three datasets test three fill + * conversion paths: small to large, large to small, and no conversion. + * They depend on `short' being smaller than `long'. + */ + /* 2. Small to large fill conversion */ #ifndef NO_FILLING - if (H5Pset_fill_value(dcpl, H5T_NATIVE_SHORT, &fill_s)<0) goto error; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_SHORT, &fill_s)<0) goto error; #endif - if ((dset1=H5Dcreate(file, "dset1", H5T_NATIVE_LONG, space, dcpl))<0) - goto error; + if ((dset1=H5Dcreate(file, "dset1", H5T_NATIVE_LONG, space, dcpl))<0) + goto error; - /* 3. Large to small fill conversion */ + /* 3. Large to small fill conversion */ #ifndef NO_FILLING - if (H5Pset_fill_value(dcpl, H5T_NATIVE_LONG, &fill_l)<0) goto error; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_LONG, &fill_l)<0) goto error; #endif - if ((dset2=H5Dcreate(file, "dset2", H5T_NATIVE_SHORT, space, dcpl))<0) - goto error; + if ((dset2=H5Dcreate(file, "dset2", H5T_NATIVE_SHORT, space, dcpl))<0) + goto error; - /* 4. No conversion */ + /* 4. No conversion */ #ifndef NO_FILLING - if (H5Pset_fill_value(dcpl, H5T_NATIVE_LONG, &fill_l)<0) goto error; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_LONG, &fill_l)<0) goto error; #endif - if ((dset3=H5Dcreate(file, "dset3", H5T_NATIVE_LONG, space, dcpl))<0) - goto error; - - /* 5. late space allocation and never write fill value */ - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; - if ((dset4=H5Dcreate(file, "dset4", H5T_NATIVE_LONG, space, dcpl))<0) - goto error; + if ((dset3=H5Dcreate(file, "dset3", H5T_NATIVE_LONG, space, dcpl))<0) + goto error; - /* 6. fill value is undefined while fill write time is H5D_FILL_TIME_ALLOC. - * Supposed to fail. */ - if(H5Pset_fill_value(dcpl, -1, NULL)<0) goto error; - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; - H5E_BEGIN_TRY { - if(H5Dcreate(file, "dset7", H5T_NATIVE_LONG, space, dcpl)!=FAIL) + /* 5. late space allocation and never write fill value */ + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; + if ((dset4=H5Dcreate(file, "dset4", H5T_NATIVE_LONG, space, dcpl))<0) goto error; - } H5E_END_TRY; - - - + /* 6. fill value is undefined while fill write time is H5D_FILL_TIME_ALLOC. + * Supposed to fail. */ + if(H5Pset_fill_value(dcpl, -1, NULL)<0) goto error; + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; + H5E_BEGIN_TRY { + if(H5Dcreate(file, "dset7", H5T_NATIVE_LONG, space, dcpl)!=FAIL) + goto error; + } H5E_END_TRY; + } + /* II. Test early space allocation cases */ + if (H5Pclose(dcpl)<0) goto error; if ((dcpl=H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5D_CHUNKED==layout) { if (H5Pset_chunk(dcpl, 5, ch_size)<0) goto error; + } else if (H5D_COMPACT==layout) { + if (H5Pset_layout(dcpl, H5D_COMPACT)<0) goto error; } if(H5Pset_space_time(dcpl, H5D_SPACE_ALLOC_EARLY) < 0) goto error; @@ -345,16 +355,17 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) goto error; } H5E_END_TRY; - /* Close everything */ - if (H5Dclose(dset1)<0) goto error; - if (H5Dclose(dset2)<0) goto error; - if (H5Dclose(dset3)<0) goto error; - if (H5Dclose(dset4)<0) goto error; + if(H5D_COMPACT != layout) { + if (H5Dclose(dset1)<0) goto error; + if (H5Dclose(dset2)<0) goto error; + if (H5Dclose(dset3)<0) goto error; + if (H5Dclose(dset4)<0) goto error; + if (H5Dclose(dset9)<0) goto error; + } if (H5Dclose(dset5)<0) goto error; if (H5Dclose(dset6)<0) goto error; if (H5Dclose(dset8)<0) goto error; - if (H5Dclose(dset9)<0) goto error; if (H5Sclose(space)<0) goto error; if (H5Pclose(dcpl)<0) goto error; if (H5Fclose(file)<0) goto error; @@ -363,101 +374,114 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) if ((file=H5Fopen(filename, H5F_ACC_RDONLY, fapl))<0) goto error; - /* I. Test cases for late space allocation */ - - /* 1. Large to small conversion */ - if ((dset1=H5Dopen(file, "dset1"))<0) goto error; - if ((dcpl=H5Dget_create_plist(dset1))<0) goto error; + /* I. Check cases for late space allocation except compact dataset */ + if(H5D_COMPACT != layout) { + /* 1. Large to small conversion */ + if ((dset1=H5Dopen(file, "dset1"))<0) goto error; + if ((dcpl=H5Dget_create_plist(dset1))<0) goto error; #ifndef NO_FILLING - if (H5Pget_fill_value(dcpl, H5T_NATIVE_SHORT, &rd_s)<0) goto error; - if (rd_s!=fill_s) { - H5_FAILED(); - printf(" %d: Got a different fill value than what was set.",__LINE__); - printf(" Got %d, set %d\n", rd_s, fill_s); - goto error; - } + if (H5Pget_fill_value(dcpl, H5T_NATIVE_SHORT, &rd_s)<0) goto error; + if (rd_s!=fill_s) { + H5_FAILED(); + printf(" %d: Got a different fill value than what was set.",__LINE__); + printf(" Got %d, set %d\n", rd_s, fill_s); + goto error; + } #endif - if (H5Dclose(dset1)<0) goto error; - if (H5Pclose(dcpl)<0) goto error; + if (H5Dclose(dset1)<0) goto error; + if (H5Pclose(dcpl)<0) goto error; - /* 2. Small to large conversion */ - if ((dset2=H5Dopen(file, "dset2"))<0) goto error; - if ((dcpl=H5Dget_create_plist(dset2))<0) goto error; + /* 2. Small to large conversion */ + if ((dset2=H5Dopen(file, "dset2"))<0) goto error; + if ((dcpl=H5Dget_create_plist(dset2))<0) goto error; #ifndef NO_FILLING - if (H5Pget_fill_value(dcpl, H5T_NATIVE_LONG, &rd_l)<0) goto error; - if (rd_l!=fill_l) { - H5_FAILED(); - printf(" %d: Got a different fill value than what was set.",__LINE__); - printf(" Got %ld, set %ld\n", rd_l, fill_l); - goto error; - } + if (H5Pget_fill_value(dcpl, H5T_NATIVE_LONG, &rd_l)<0) goto error; + if (rd_l!=fill_l) { + H5_FAILED(); + printf(" %d: Got a different fill value than what was set.",__LINE__); + printf(" Got %ld, set %ld\n", rd_l, fill_l); + goto error; + } #endif - if (H5Dclose(dset2)<0) goto error; - if (H5Pclose(dcpl)<0) goto error; + if (H5Dclose(dset2)<0) goto error; + if (H5Pclose(dcpl)<0) goto error; - /* 3. No conversion */ - if ((dset3=H5Dopen(file, "dset3"))<0) goto error; - if ((dcpl=H5Dget_create_plist(dset3))<0) goto error; + /* 3. No conversion */ + if ((dset3=H5Dopen(file, "dset3"))<0) goto error; + if ((dcpl=H5Dget_create_plist(dset3))<0) goto error; #ifndef NO_FILLING - if (H5Pget_fill_value(dcpl, H5T_NATIVE_LONG, &rd_l)<0) goto error; - if (rd_l!=fill_l) { - H5_FAILED(); - printf(" %d: Got a different fill value than what was set.",__LINE__); - printf(" Got %ld, set %ld\n", rd_l, fill_l); - goto error; - } + if (H5Pget_fill_value(dcpl, H5T_NATIVE_LONG, &rd_l)<0) goto error; + if (rd_l!=fill_l) { + H5_FAILED(); + printf(" %d: Got a different fill value than what was set.",__LINE__); + printf(" Got %ld, set %ld\n", rd_l, fill_l); + goto error; + } #endif - if(H5Pget_space_time(dcpl, &alloc_time)<0) goto error; - if(H5Pget_fill_time(dcpl, &fill_time)<0) goto error; - if(alloc_time != H5D_SPACE_ALLOC_LATE) { - H5_FAILED(); - puts(" Got non-H5D_SPACE_ALLOC_LATE space allocation time."); - printf(" Got %d\n", alloc_time); - } - if(fill_time != H5D_FILL_TIME_ALLOC) { - H5_FAILED(); - puts(" Got non-H5D_FILL_TIME_ALLOC fill value write time."); - printf(" Got %d\n", fill_time); - } - if (H5Dclose(dset3)<0) goto error; - if (H5Pclose(dcpl)<0) goto error; + if(H5Pget_space_time(dcpl, &alloc_time)<0) goto error; + if(H5Pget_fill_time(dcpl, &fill_time)<0) goto error; + if(alloc_time != H5D_SPACE_ALLOC_LATE) { + H5_FAILED(); + puts(" Got non-H5D_SPACE_ALLOC_LATE space allocation time."); + printf(" Got %d\n", alloc_time); + } + if(fill_time != H5D_FILL_TIME_ALLOC) { + H5_FAILED(); + puts(" Got non-H5D_FILL_TIME_ALLOC fill value write time."); + printf(" Got %d\n", fill_time); + } + if (H5Dclose(dset3)<0) goto error; + if (H5Pclose(dcpl)<0) goto error; - /* 4. late space allocation and never write fill value */ - if ((dset4=H5Dopen(file, "dset4"))<0) goto error; - if (H5Dget_space_status(dset4, &allocation)<0) goto error; + /* 4. late space allocation and never write fill value */ + if ((dset4=H5Dopen(file, "dset4"))<0) goto error; + if (H5Dget_space_status(dset4, &allocation)<0) goto error; #ifndef H5_HAVE_PARALLEL - if (layout == H5D_CONTIGUOUS && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED) { - H5_FAILED(); - puts(" Got allocated space instead of unallocated."); - printf(" Got %d\n", allocation); - goto error; - } + if (layout == H5D_CONTIGUOUS && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED) { + H5_FAILED(); + puts(" Got allocated space instead of unallocated."); + printf(" Got %d\n", allocation); + goto error; + } #else - if (layout == H5D_CONTIGUOUS && allocation == H5D_SPACE_STATUS_NOT_ALLOCATED) { - H5_FAILED(); - printf(" %d: Got unallocated space instead of allocated.\n",__LINE__); - printf(" Got %d\n", allocation); - goto error; - } + if (layout == H5D_CONTIGUOUS && allocation == H5D_SPACE_STATUS_NOT_ALLOCATED) { + H5_FAILED(); + printf(" %d: Got unallocated space instead of allocated.\n",__LINE__); + printf(" Got %d\n", allocation); + goto error; + } #endif - if ((dcpl=H5Dget_create_plist(dset4))<0) goto error; - if(H5Pget_space_time(dcpl, &alloc_time)<0) goto error; - if(H5Pget_fill_time(dcpl, &fill_time)<0) goto error; - if(alloc_time != H5D_SPACE_ALLOC_LATE) { - H5_FAILED(); - puts(" Got non-H5D_SPACE_ALLOC_LATE space allocation time."); - printf(" Got %d\n", alloc_time); - } - if(fill_time != H5D_FILL_TIME_NEVER) { - H5_FAILED(); - puts(" Got non-H5D_FILL_TIME_NEVER fill value write time."); - printf(" Got %d\n", fill_time); + if ((dcpl=H5Dget_create_plist(dset4))<0) goto error; + if(H5Pget_space_time(dcpl, &alloc_time)<0) goto error; + if(H5Pget_fill_time(dcpl, &fill_time)<0) goto error; + if(alloc_time != H5D_SPACE_ALLOC_LATE) { + H5_FAILED(); + puts(" Got non-H5D_SPACE_ALLOC_LATE space allocation time."); + printf(" Got %d\n", alloc_time); + } + if(fill_time != H5D_FILL_TIME_NEVER) { + H5_FAILED(); + puts(" Got non-H5D_FILL_TIME_NEVER fill value write time."); + printf(" Got %d\n", fill_time); + } + if (H5Dclose(dset4)<0) goto error; + if (H5Pclose(dcpl)<0) goto error; + + /* 5. Compound datatype test */ + if ((dset9=H5Dopen(file, "dset9"))<0) goto error; + if ((dcpl=H5Dget_create_plist(dset9))<0) goto error; + if (H5Pget_fill_value(dcpl, comp_type_id, &rd_c)<0) goto error; + if( rd_c.a!=0 || rd_c.y != fill_ctype.y || rd_c.x != 0 || rd_c.z != '\0') { + H5_FAILED(); + puts(" Got wrong fill value"); + printf(" Got rd_c.a=%f, rd_c.y=%f and rd_c.x=%d, rd_c.z=%c\n", + rd_c.a, rd_c.y, rd_c.x, rd_c.z); + } + if (H5Dclose(dset9)<0) goto error; + if (H5Pclose(dcpl)<0) goto error; } - if (H5Dclose(dset4)<0) goto error; - if (H5Pclose(dcpl)<0) goto error; - - /* II. Check early space allocation case */ + /* II. Check early space allocation cases */ /* 1. Never write fill value */ if ((dset5=H5Dopen(file, "dset5"))<0) goto error; @@ -529,21 +553,6 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) if (H5Dclose(dset8)<0) goto error; if (H5Pclose(dcpl)<0) goto error; - /* 4. Compound datatype test */ - if ((dset9=H5Dopen(file, "dset9"))<0) goto error; - if ((dcpl=H5Dget_create_plist(dset9))<0) goto error; - if (H5Pget_fill_value(dcpl, comp_type_id, &rd_c)<0) goto error; - if( rd_c.a!=0 || rd_c.y != fill_ctype.y || rd_c.x != 0 || rd_c.z != '\0') { - H5_FAILED(); - puts(" Got wrong fill value"); - printf(" Got rd_c.a=%f, rd_c.y=%f and rd_c.x=%d, rd_c.z=%c\n", - rd_c.a, rd_c.y, rd_c.x, rd_c.z); - } - if (H5Dclose(dset9)<0) goto error; - if (H5Pclose(dcpl)<0) goto error; - - - if (H5Tclose(comp_type_id)<0) goto error; if (H5Fclose(file)<0) goto error; PASSED(); return 0; @@ -552,14 +561,16 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) H5E_BEGIN_TRY { H5Pclose(dcpl); H5Sclose(space); - H5Dclose(dset1); - H5Dclose(dset2); - H5Dclose(dset3); - H5Dclose(dset4); + if(H5D_COMPACT != layout) { + H5Dclose(dset1); + H5Dclose(dset2); + H5Dclose(dset3); + H5Dclose(dset4); + H5Dclose(dset9); + } H5Dclose(dset5); H5Dclose(dset6); H5Dclose(dset8); - H5Dclose(dset9); H5Fclose(file); } H5E_END_TRY; return 1; @@ -589,7 +600,7 @@ test_rdwr_cases(hid_t file, hid_t dcpl, const char *dname, void *_fillval, H5T_class_t datatype, hid_t ctype_id) { hid_t fspace=-1, mspace=-1, dset1=-1, dset2=-1; - hsize_t cur_size[5] = {32, 16, 8, 4, 2}; + hsize_t cur_size[5] = {2, 16, 8, 4, 2}; hsize_t one[5] = {1, 1, 1, 1, 1}; hsize_t hs_size[5], hs_stride[5]; hssize_t hs_offset[5], nelmts; @@ -819,6 +830,7 @@ test_rdwr_cases(hid_t file, hid_t dcpl, const char *dname, void *_fillval, H5Sclose(fspace); H5Sclose(mspace); } H5E_END_TRY; + return 1; } @@ -846,13 +858,15 @@ test_rdwr(hid_t fapl, const char *base_name, H5D_layout_t layout) { char filename[1024]; hid_t file=-1, dcpl=-1, ctype_id=-1; - hsize_t ch_size[5] = {1, 16, 8, 4, 2}; + hsize_t ch_size[5] = {2, 16, 8, 4, 2}; int nerrors=0; int fillval = 0x4c70f1cd; comp_datatype fill_ctype={0,0,0,0}; if (H5D_CHUNKED==layout) { TESTING("chunked dataset I/O"); + } else if (H5D_COMPACT==layout) { + TESTING("compact dataset I/O"); } else { TESTING("contiguous dataset I/O"); } @@ -864,62 +878,65 @@ test_rdwr(hid_t fapl, const char *base_name, H5D_layout_t layout) if ((dcpl=H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5D_CHUNKED==layout) { if (H5Pset_chunk(dcpl, 5, ch_size)<0) goto error; + } else if (H5D_COMPACT==layout) { + if (H5Pset_layout(dcpl, H5D_COMPACT)<0) goto error; } if ((ctype_id=create_compound_type())<0) goto error; /* I. Test H5D_SPACE_ALLOC_LATE space allocation cases */ - if(H5Pset_space_time(dcpl, H5D_SPACE_ALLOC_LATE) < 0) goto error; - - /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value to be default */ - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; - fillval = 0; - nerrors += test_rdwr_cases(file, dcpl, "dset1", &fillval, H5D_FILL_TIME_ALLOC, + if(H5D_COMPACT != layout) { + if(H5Pset_space_time(dcpl, H5D_SPACE_ALLOC_LATE) < 0) goto error; + + /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value to be default */ + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; + fillval = 0; + nerrors += test_rdwr_cases(file, dcpl, "dset1", &fillval, H5D_FILL_TIME_ALLOC, + layout, H5T_INTEGER, -1); + + /* case for H5D_FILL_TIME_NEVER as fill write time and fill value to be default */ + if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; + nerrors += test_rdwr_cases(file, dcpl, "dset2", &fillval, H5D_FILL_TIME_NEVER, layout, H5T_INTEGER, -1); - /* case for H5D_FILL_TIME_NEVER as fill write time and fill value to be default */ - if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; - nerrors += test_rdwr_cases(file, dcpl, "dset2", &fillval, H5D_FILL_TIME_NEVER, + /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value is user-defined */ + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; + fillval = 0x4c70f1cd; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval)<0) goto error; + nerrors += test_rdwr_cases(file, dcpl, "dset3", &fillval, H5D_FILL_TIME_ALLOC, layout, H5T_INTEGER, -1); - /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value is user-defined */ - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; - fillval = 0x4c70f1cd; - if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval)<0) goto error; - nerrors += test_rdwr_cases(file, dcpl, "dset3", &fillval, H5D_FILL_TIME_ALLOC, + /* case for H5D_FILL_TIME_NEVER as fill write time and fill value is user-defined */ + if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval)<0) goto error; + nerrors += test_rdwr_cases(file, dcpl, "dset4", &fillval, H5D_FILL_TIME_NEVER, layout, H5T_INTEGER, -1); - /* case for H5D_FILL_TIME_NEVER as fill write time and fill value is user-defined */ - if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; - if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval)<0) goto error; - nerrors += test_rdwr_cases(file, dcpl, "dset4", &fillval, H5D_FILL_TIME_NEVER, - layout, H5T_INTEGER, -1); + /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value is undefined */ + /* This case has been tested in test_create() function */ - /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value is undefined */ - /* This case has been tested in test_create() function */ - - /* case for H5D_FILL_TIME_NEVER as fill write time and fill value is undefined */ - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; - if (H5Pset_fill_value(dcpl, -1, NULL)<0) goto error; - nerrors += test_rdwr_cases(file, dcpl, "dset5", &fillval, H5D_FILL_TIME_NEVER, + /* case for H5D_FILL_TIME_NEVER as fill write time and fill value is undefined */ + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; + if (H5Pset_fill_value(dcpl, -1, NULL)<0) goto error; + nerrors += test_rdwr_cases(file, dcpl, "dset5", &fillval, H5D_FILL_TIME_NEVER, layout, H5T_INTEGER, -1); - /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value is user-defined - * as compound type */ - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; - fill_ctype.y = 4444.4444; - if(H5Pset_fill_value(dcpl, ctype_id, &fill_ctype)<0) goto error; - nerrors += test_rdwr_cases(file, dcpl, "dset11", &fill_ctype, H5D_FILL_TIME_ALLOC, + /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value is user-defined + * as compound type */ + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; + fill_ctype.y = 4444.4444; + if(H5Pset_fill_value(dcpl, ctype_id, &fill_ctype)<0) goto error; + nerrors += test_rdwr_cases(file, dcpl, "dset11", &fill_ctype, H5D_FILL_TIME_ALLOC, layout, H5T_COMPOUND, ctype_id); - if (H5Pclose(dcpl)<0) goto error; - if ((dcpl=H5Pcreate(H5P_DATASET_CREATE))<0) goto error; - if (H5D_CHUNKED==layout) { - if (H5Pset_chunk(dcpl, 5, ch_size)<0) goto error; + if (H5Pclose(dcpl)<0) goto error; + if ((dcpl=H5Pcreate(H5P_DATASET_CREATE))<0) goto error; + if (H5D_CHUNKED==layout) { + if (H5Pset_chunk(dcpl, 5, ch_size)<0) goto error; + } } - /* II. Test H5D_SPACE_ALLOC_EARLY space allocation cases */ if(H5Pset_space_time(dcpl, H5D_SPACE_ALLOC_EARLY) < 0) goto error; @@ -932,7 +949,7 @@ test_rdwr(hid_t fapl, const char *base_name, H5D_layout_t layout) /* case for H5D_FILL_TIME_NEVER as fill write time and fill value to be default */ if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) goto error; nerrors += test_rdwr_cases(file, dcpl, "dset7", &fillval, H5D_FILL_TIME_NEVER, layout, - H5T_INTEGER, -1); + H5T_INTEGER, -1); /* case for H5D_FILL_TIME_ALLOC as fill write time and fill value is user-defined */ if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; @@ -1381,18 +1398,20 @@ error: int main(int argc, char *argv[]) { - int nerrors=0, argno, test_contig=1, test_chunk=1; + int nerrors=0, argno, test_contig=1, test_chunk=1, test_compact=1; hid_t fapl=-1; if (argc>=2) { - test_contig = test_chunk = 0; + test_contig = test_chunk = test_compact = 0; for (argno=1; argno<argc; argno++) { if (!strcmp(argv[argno], "contiguous")) { test_contig = 1; } else if (!strcmp(argv[argno], "chunked")) { test_chunk = 1; - } else { - fprintf(stderr, "usage: %s [contiguous] [chunked]\n", argv[0]); + } else if (!strcmp(argv[argno], "compact")) { + test_compact =1; + } else { + fprintf(stderr, "usage: %s [contiguous] [chunked] [compact]\n", argv[0]); exit(1); } } @@ -1417,7 +1436,13 @@ main(int argc, char *argv[]) nerrors += test_extend(fapl, FILENAME[5], H5D_CONTIGUOUS); nerrors += test_compatible(); } - + + /* Compact dataset storage tests */ + if (test_compact) { + nerrors += test_create(fapl, FILENAME[6], H5D_COMPACT); + nerrors += test_rdwr (fapl, FILENAME[7], H5D_COMPACT); + } + if (nerrors) goto error; puts("All fill value tests passed."); if (h5_cleanup(FILENAME, fapl)) remove(FILE_NAME_RAW); diff --git a/test/tmisc.c b/test/tmisc.c index e0b3d92..7f6eaa1 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -127,6 +127,7 @@ typedef struct #define MISC8_DSETNAME4 "Dataset4" #define MISC8_DSETNAME5 "Dataset5" #define MISC8_DSETNAME6 "Dataset6" +#define MISC8_DSETNAME7 "Dataset7" #define MISC8_RANK 2 #define MISC8_DIM0 100 #define MISC8_DIM1 100 @@ -1114,6 +1115,11 @@ test_misc8(void) dcpl = H5Pcreate(H5P_DATASET_CREATE); CHECK(dcpl, FAIL, "H5Pcreate"); + /* I. contiguous dataset tests */ + + ret = H5Pset_layout(dcpl, H5D_CONTIGUOUS); + CHECK(ret, FAIL, "H5Pset_layout"); + /* Set the space allocation time to early */ ret = H5Pset_space_time(dcpl,H5D_SPACE_ALLOC_EARLY); CHECK(ret, FAIL, "H5Pset_space_time"); @@ -1158,6 +1164,33 @@ test_misc8(void) CHECK(ret, FAIL, "H5Dclose"); #endif /* H5_HAVE_PARALLEL */ + /* II. compact dataset tests */ + ret = H5Pset_layout(dcpl, H5D_COMPACT); + CHECK(ret, FAIL, "H5Pset_layout"); + + /* Set the space allocation time to early */ + ret = H5Pset_space_time(dcpl,H5D_SPACE_ALLOC_EARLY); + CHECK(ret, FAIL, "H5Pset_space_time"); + + /* Create a contiguous dataset, with space allocation early */ + did = H5Dcreate(fid, MISC8_DSETNAME7, H5T_NATIVE_INT, sid, dcpl); + CHECK(did, FAIL, "H5Dcreate"); + + /* Check the storage size */ + storage_size=H5Dget_storage_size(did); + CHECK(storage_size, 0, "H5Dget_storage_size"); + VERIFY(storage_size, MISC8_DIM0*MISC8_DIM1*H5Tget_size(H5T_NATIVE_INT), "H5Dget_storage_size"); + + /* Close dataset ID */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + + + /* III. chunked dataset tests */ + + ret = H5Pset_layout(dcpl, H5D_CHUNKED); + CHECK(ret, FAIL, "H5Pset_layout"); + /* Set the space allocation time to early */ ret = H5Pset_space_time(dcpl,H5D_SPACE_ALLOC_EARLY); CHECK(ret, FAIL, "H5Pset_space_time"); @@ -1271,10 +1304,17 @@ test_misc8(void) /* Check the storage size after only four chunks are written */ storage_size=H5Dget_storage_size(did); CHECK(storage_size, 0, "H5Dget_storage_size"); +#ifdef H5_HAVE_COMPRESSION if(storage_size>=(4*MISC8_CHUNK_DIM0*MISC8_CHUNK_DIM1*H5Tget_size(H5T_NATIVE_INT))) { num_errs++; printf("Error on line %d: data wasn't compressed! storage_size=%u\n",__LINE__,(unsigned)storage_size); } +#else /* Compression is not configured */ + if(storage_size!=(4*MISC8_CHUNK_DIM0*MISC8_CHUNK_DIM1*H5Tget_size(H5T_NATIVE_INT))) { + num_errs++; + printf("Error on line %d: wrong storage size! storage_size=%u\n",__LINE__,(unsigned)storage_size); + } +#endif /* H5_HAVE_COMPRESSION */ /* Write entire dataset */ ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata); @@ -1299,10 +1339,17 @@ test_misc8(void) /* Check the storage size after data is written */ storage_size=H5Dget_storage_size(did); CHECK(storage_size, 0, "H5Dget_storage_size"); +#ifdef H5_HAVE_COMPRESSION if(storage_size>=(MISC8_DIM0*MISC8_DIM1*H5Tget_size(H5T_NATIVE_INT))) { num_errs++; printf("Error on line %d: data wasn't compressed! storage_size=%u\n",__LINE__,(unsigned)storage_size); } +#else + if(storage_size!=(MISC8_DIM0*MISC8_DIM1*H5Tget_size(H5T_NATIVE_INT))) { + num_errs++; + printf("Error on line %d: wrong storage size! storage_size=%u\n",__LINE__,(unsigned)storage_size); + } +#endif /*H5_HAVE_COMPRESSION*/ /* Close dataset ID */ ret = H5Dclose(did); diff --git a/testpar/t_mdset.c b/testpar/t_mdset.c index 0a97e5e..c03de3c 100644 --- a/testpar/t_mdset.c +++ b/testpar/t_mdset.c @@ -80,6 +80,117 @@ void multiple_dset_write(char *filename, int ndatasets) H5Fclose (iof); } +/* Example of using PHDF5 to create, write, and read compact dataset. + * Hyperslab is prohibited for write. + */ +void compact_dataset(char *filename) +{ + int i, j, n, mpi_size, mpi_rank, err_num=0; + hid_t iof, plist, dcpl, dxpl, dataset, memspace, filespace; + hssize_t chunk_origin [DIM]; + hsize_t chunk_dims [DIM], file_dims [DIM]; + hsize_t count[DIM]={1,1}; + double outme [SIZE][SIZE], inme[SIZE][SIZE]; + char dname[]="dataset"; + herr_t ret; + + MPI_Comm_rank (MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size (MPI_COMM_WORLD, &mpi_size); + + VRFY((mpi_size <= SIZE), "mpi_size <= SIZE"); + + chunk_origin [0] = mpi_rank * (SIZE / mpi_size); + chunk_origin [1] = 0; + chunk_dims [0] = SIZE / mpi_size; + chunk_dims [1] = SIZE; + + for (i = 0; i < DIM; i++) + file_dims [i] = SIZE; + + plist = create_faccess_plist(MPI_COMM_WORLD, MPI_INFO_NULL, facc_type); + iof = H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, plist); + + /* Define data space */ + memspace = H5Screate_simple (DIM, chunk_dims, NULL); + filespace = H5Screate_simple (DIM, file_dims, NULL); + + /* Create a compact dataset */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + VRFY((dcpl>=0), "dataset creation property list succeeded"); + ret=H5Pset_layout(dcpl, H5D_COMPACT); + VRFY((dcpl >= 0), "set property list for compact dataset"); + ret=H5Pset_space_time(dcpl, H5D_SPACE_ALLOC_EARLY); + VRFY((ret >= 0), "set space allocation time for compact dataset"); + + dataset = H5Dcreate (iof, dname, H5T_NATIVE_DOUBLE, filespace, dcpl); + VRFY((dataset >= 0), "H5Dcreate succeeded"); + + /* Define hyperslab */ + ret = H5Sselect_hyperslab (filespace, H5S_SELECT_SET, chunk_origin, chunk_dims, count, chunk_dims); + VRFY((ret>=0), "mdata hyperslab selection"); + + /* set up the collective transfer properties list */ + dxpl = H5Pcreate (H5P_DATASET_XFER); + VRFY((dxpl >= 0), ""); + ret=H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + + /* calculate data to write */ + for (i = 0; i < SIZE; i++) + for (j = 0; j < SIZE; j++) + outme [i][j] = (i+j)*1000 + mpi_rank; + + /* Test hyperslab writing. Supposed to fail */ + H5E_BEGIN_TRY { + ret=H5Dwrite(dataset, H5T_NATIVE_DOUBLE, memspace, filespace, dxpl, outme); + } H5E_END_TRY; + VRFY((ret < 0), "H5Dwrite hyperslab write failed as expected"); + + /* Recalculate data to write. Each process writes the same data. */ + for (i = 0; i < SIZE; i++) + for (j = 0; j < SIZE; j++) + outme [i][j] = (i+j)*1000; + + ret=H5Dwrite (dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, dxpl, outme); + VRFY((ret >= 0), "H5Dwrite succeeded"); + + H5Pclose (dcpl); + H5Pclose (plist); + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Fclose (iof); + + /* Open the file and dataset, read and compare the data. */ + plist = create_faccess_plist(MPI_COMM_WORLD, MPI_INFO_NULL, facc_type); + iof = H5Fopen(filename, H5F_ACC_RDONLY, plist); + VRFY((iof >= 0), "H5Fopen succeeded"); + + /* set up the collective transfer properties list */ + dxpl = H5Pcreate (H5P_DATASET_XFER); + VRFY((dxpl >= 0), ""); + ret=H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + + dataset = H5Dopen(iof, dname); + VRFY((dataset >= 0), "H5Dcreate succeeded"); + + ret = H5Dread(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, dxpl, inme); + VRFY((ret >= 0), "H5Dread succeeded"); + + /* Verify data value */ + for (i = 0; i < SIZE; i++) + for (j = 0; j < SIZE; j++) + if(inme[i][j] != outme[i][j]) + if(err_num++ < MAX_ERR_REPORT || verbose) + printf("Dataset Verify failed at [%d][%d]: expect %f, got %f\n", i, j, outme[i][j], inme[i][j]); + + H5Pclose(plist); + H5Pclose(dxpl); + H5Dclose(dataset); + H5Fclose(iof); +} + /* * Example of using PHDF5 to create multiple groups. Under the root group, * it creates ngroups groups. Under the first group just created, it creates diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index b650e00..95fe93c 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -28,15 +28,17 @@ void *old_client_data; /* previous error handler arg.*/ /* other option flags */ int doread=1; /* read test */ int dowrite=1; /* write test */ +int docompact=1; /* compact dataset test */ /* FILENAME and filenames must have the same number of names */ -const char *FILENAME[6]={ +const char *FILENAME[7]={ "ParaEg1", "ParaEg2", "ParaEg3", "ParaMdset", "ParaMgroup", + "ParaCompact", NULL}; -char filenames[6][PATH_MAX]; +char filenames[7][PATH_MAX]; hid_t fapl; /* file access property list */ #ifdef USE_PAUSE @@ -97,13 +99,14 @@ static void usage(void) { printf("Usage: testphdf5 [-r] [-w] [-v] [-m<n_datasets>] [-n<n_groups>] " - "[-f <prefix>] [-d <dim0> <dim1>]\n"); + "[-o] [-f <prefix>] [-d <dim0> <dim1>]\n"); printf("\t-r\t\tno read test\n"); printf("\t-w\t\tno write test\n"); printf("\t-m<n_datasets>" "\tset number of datasets for the multiple dataset test\n"); printf("\t-n<n_groups>" "\tset number of groups for the multiple group test\n"); + printf("\t-o\t\tno compact dataset test\n"); printf("\t-v\t\tverbose on\n"); printf("\t-f <prefix>\tfilename prefix\n"); printf("\t-s\t\tuse Split-file together with MPIO\n"); @@ -152,6 +155,8 @@ parse_options(int argc, char **argv) return(1); } break; + case 'o': docompact = 0; + break; case 'v': verbose = 1; break; case 'f': if (--argc < 1) { @@ -380,7 +385,15 @@ int main(int argc, char **argv) MPI_BANNER("read tests skipped"); } - if (!(dowrite || doread || ndatasets || ngroups)){ + if (docompact){ + MPI_BANNER("compact dataset test..."); + compact_dataset(filenames[5]); + } + else { + MPI_BANNER("compact dataset test skipped"); + } + + if (!(dowrite || doread || ndatasets || ngroups || docompact)){ usage(); nerrors++; } |