diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5D.c | 12 | ||||
| -rw-r--r-- | src/H5Distore.c | 94 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 8 | ||||
| -rw-r--r-- | src/H5Fistore.c | 94 | ||||
| -rw-r--r-- | src/H5Pdcpl.c | 42 | ||||
| -rw-r--r-- | src/H5Pdxpl.c | 125 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 5 | ||||
| -rw-r--r-- | src/H5Tconv.c | 2 | ||||
| -rw-r--r-- | src/H5Tvlen.c | 10 | ||||
| -rw-r--r-- | src/H5Z.c | 71 | ||||
| -rw-r--r-- | src/H5Zadler32.c | 139 | ||||
| -rw-r--r-- | src/H5Zdeflate.c | 10 | ||||
| -rw-r--r-- | src/H5Zprivate.h | 11 | ||||
| -rw-r--r-- | src/H5Zpublic.h | 31 | ||||
| -rw-r--r-- | src/H5Zshuffle.c | 23 | ||||
| -rw-r--r-- | src/H5config.h.in | 3 | ||||
| -rw-r--r-- | src/Makefile.in | 2 |
17 files changed, 577 insertions, 105 deletions
@@ -167,6 +167,8 @@ H5D_init_interface(void) * - Default value for file driver info * - Default value for 'gather reads' property * - Default value for vector size + * - Default value for EDC property + * - Default value for filter callback */ H5P_genclass_t *xfer_pclass; size_t def_max_temp_buf = H5D_XFER_MAX_TEMP_BUF_DEF; @@ -185,6 +187,8 @@ H5D_init_interface(void) hid_t def_vfl_id = H5D_XFER_VFL_ID_DEF; void *def_vfl_info = H5D_XFER_VFL_INFO_DEF; size_t def_hyp_vec_size = H5D_XFER_HYPER_VECTOR_SIZE_DEF; + H5Z_EDC_t enable_edc = H5D_XFER_EDC_DEF; + H5Z_cb_t filter_cb = H5D_XFER_FILTER_CB_DEF; /* Dataset creation property class variables. In sequence, they are, * - Creation property list class to modify @@ -286,6 +290,14 @@ H5D_init_interface(void) /* Register the vector size property */ if(H5P_register(xfer_pclass,H5D_XFER_HYPER_VECTOR_SIZE_NAME,H5D_XFER_HYPER_VECTOR_SIZE_SIZE,&def_hyp_vec_size,NULL,NULL,NULL,NULL,NULL,NULL)<0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + + /* Register the EDC property */ + if(H5P_register(xfer_pclass,H5D_XFER_EDC_NAME,H5D_XFER_EDC_SIZE,&enable_edc,NULL,NULL,NULL,NULL,NULL,NULL)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); + + /* Register the filter callback property */ + if(H5P_register(xfer_pclass,H5D_XFER_FILTER_CB_NAME,H5D_XFER_FILTER_CB_SIZE,&filter_cb,NULL,NULL,NULL,NULL,NULL,NULL)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class"); } /* end if */ /* Only register the default property list if it hasn't been created yet */ diff --git a/src/H5Distore.c b/src/H5Distore.c index aa88704..eefa690 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -898,12 +898,15 @@ H5F_istore_flush_entry(H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t reset) void *buf=NULL; /*temporary buffer */ size_t alloc; /*bytes allocated for BUF */ hbool_t point_of_no_return = FALSE; - + H5Z_cb_t cb_struct={NULL,NULL}; + H5Z_EDC_t edc=H5Z_ENABLE_EDC; + FUNC_ENTER_NOINIT(H5F_istore_flush_entry); assert(f); assert(ent); assert(!ent->locked); + HDmemset(&udata, 0, sizeof(H5F_istore_ud1_t)); buf = ent->chunk; if (ent->dirty) { @@ -941,8 +944,11 @@ H5F_istore_flush_entry(H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t reset) point_of_no_return = TRUE; ent->chunk = NULL; } - if (H5Z_pipeline(f, ent->pline, 0, &(udata.key.filter_mask), - &(udata.key.nbytes), &alloc, &buf)<0) { + /* Don't know whether we should involve transfer property list. So + * just pass in H5Z_ENABLE_EDC and default callback setting for data + * read. */ + if (H5Z_pipeline(f, ent->pline, 0, &(udata.key.filter_mask), edc, + cb_struct, &(udata.key.nbytes), &alloc, &buf)<0) { HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed"); } @@ -1322,13 +1328,19 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, H5F_istore_ud1_t udata; /*B-tree pass-through */ size_t chunk_size=0; /*size of a chunk */ hsize_t tempchunk_size; - size_t chunk_alloc=0; /*allocated chunk size */ herr_t status; /*func return status */ void *chunk=NULL; /*the file chunk */ void *ret_value; /*return value */ H5P_genplist_t *plist=NULL; /* Property list */ + H5Z_EDC_t edc; + H5Z_cb_t cb_struct; FUNC_ENTER_NOINIT(H5F_istore_lock); + + assert(TRUE==H5P_isa_class(dxpl_id,H5P_DATASET_XFER)); + plist=H5I_object(dxpl_id); + assert(plist!=NULL); + HDmemset(&udata, 0, sizeof(H5F_istore_ud1_t)); if (rdcc->nslots>0) { for (u=0, temp_idx=0; u<layout->ndims; u++) { @@ -1371,11 +1383,11 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, for (u=0, tempchunk_size=1; u<layout->ndims; u++) tempchunk_size *= layout->dim[u]; H5_ASSIGN_OVERFLOW(chunk_size,tempchunk_size,hsize_t,size_t); - chunk_alloc = chunk_size; - if (NULL==(chunk=H5MM_malloc (chunk_alloc))) + if (NULL==(chunk=H5MM_malloc (chunk_size))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); } else { + /* * Not in the cache. Read it from the file and count this as a miss * if it's in the file or an init if it isn't. @@ -1385,40 +1397,54 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, tempchunk_size *= layout->dim[u]; } H5_ASSIGN_OVERFLOW(chunk_size,tempchunk_size,hsize_t,size_t); - chunk_alloc = chunk_size; udata.mesg = *layout; udata.addr = HADDR_UNDEF; status = H5B_find (f, H5B_ISTORE, layout->addr, &udata); H5E_clear (); - if (NULL==(chunk = H5MM_malloc (chunk_alloc))) - HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); + if (status>=0 && H5F_addr_defined(udata.addr)) { + size_t chunk_alloc=0; /*allocated chunk size */ + /* * The chunk exists on disk. */ + /* Chunk size on disk isn't [likely] the same size as the final chunk + * size in memory, so allocate memory big enough. */ + chunk_alloc = udata.key.nbytes; + if (NULL==(chunk = H5MM_malloc (chunk_alloc))) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); if (H5F_block_read(f, H5FD_MEM_DRAW, udata.addr, udata.key.nbytes, H5P_DATASET_XFER_DEFAULT, chunk)<0) HGOTO_ERROR (H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk"); - if (H5Z_pipeline(f, pline, H5Z_FLAG_REVERSE, - &(udata.key.filter_mask), &(udata.key.nbytes), - &chunk_alloc, &chunk)<0 || udata.key.nbytes!=chunk_size) + if(H5P_get(plist,H5D_XFER_EDC_NAME,&edc)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get edc information"); + if(H5P_get(plist,H5D_XFER_FILTER_CB_NAME,&cb_struct)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get filter callback struct"); + if (H5Z_pipeline(f, pline, H5Z_FLAG_REVERSE, &(udata.key.filter_mask), edc, + cb_struct, &(udata.key.nbytes), &chunk_alloc, &chunk)<0) { HGOTO_ERROR(H5E_PLINE, H5E_READERROR, NULL, "data pipeline read failed"); + } rdcc->nmisses++; - } else if (fill && fill->buf) { - /* - * The chunk doesn't exist in the file. Replicate the fill - * value throughout the chunk. - */ - assert(0==chunk_size % fill->size); - H5V_array_fill(chunk, fill->buf, fill->size, chunk_size/fill->size); - rdcc->ninits++; } else { - /* - * The chunk doesn't exist in the file and no fill value was - * specified. Assume all zeros. - */ - HDmemset (chunk, 0, chunk_size); + /* Chunk size on disk isn't [likely] the same size as the final chunk + * size in memory, so allocate memory big enough. */ + if (NULL==(chunk = H5MM_malloc (chunk_size))) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); + if (fill && fill->buf) { + /* + * The chunk doesn't exist in the file. Replicate the fill + * value throughout the chunk. + */ + assert(0==chunk_size % fill->size); + H5V_array_fill(chunk, fill->buf, fill->size, chunk_size/fill->size); + } else { + /* + * The chunk doesn't exist in the file and no fill value was + * specified. Assume all zeros. + */ + HDmemset (chunk, 0, chunk_size); + } rdcc->ninits++; - } + } /* end else */ } assert (found || chunk_size>0); @@ -1436,6 +1462,7 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, if (H5F_istore_preempt(f, ent, TRUE)<0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache"); } + if (H5F_istore_prune(f, chunk_size)<0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache"); @@ -1453,9 +1480,6 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, ent->wr_count = chunk_size; ent->chunk = chunk; - assert(TRUE==H5P_isa_class(dxpl_id,H5P_DATASET_XFER)); - plist=H5I_object(dxpl_id); - assert(plist!=NULL); H5P_get(plist,H5D_XFER_BTREE_SPLIT_RATIO_NAME,&(ent->split_ratios)); /* Add it to the cache */ @@ -1476,7 +1500,6 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, ent->prev = NULL; } found = TRUE; - } else if (!found) { /* * The chunk is larger than the entire cache so we don't cache it. @@ -1523,7 +1546,8 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, done: if (!ret_value) - H5MM_xfree (chunk); + if(chunk) + H5MM_xfree (chunk); FUNC_LEAVE_NOAPI(ret_value); } @@ -2337,6 +2361,8 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, unsigned chunk_exists; /* Flag to indicate whether a chunk exists already */ int i; /* Local index variable */ unsigned u; /* Local index variable */ + H5Z_EDC_t edc; /* Decide whether to enable EDC for read */ + H5Z_cb_t cb_struct; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5F_istore_allocate, FAIL); @@ -2363,6 +2389,10 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset transfer property list"); if(H5P_get(dx_plist,H5D_XFER_BTREE_SPLIT_RATIO_NAME,split_ratios)<0) HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "can't get B-tree split ratios"); + if(H5P_get(dx_plist,H5D_XFER_EDC_NAME,&edc)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get edc information"); + if(H5P_get(dx_plist,H5D_XFER_FILTER_CB_NAME,&cb_struct)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get filter callback struct"); #ifdef H5_HAVE_PARALLEL /* Retrieve up MPI parameters */ @@ -2435,7 +2465,7 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, size_t nbytes=(size_t)chunk_size; /* Push the chunk through the filters */ - if (H5Z_pipeline(f, &pline, 0, &filter_mask, &nbytes, &buf_size, &chunk)<0) + if (H5Z_pipeline(f, &pline, 0, &filter_mask, edc, cb_struct, &nbytes, &buf_size, &chunk)<0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed"); /* Keep the number of bytes the chunk turned in to */ diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index cfef073..239a187 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -140,6 +140,14 @@ #define H5D_XFER_HYPER_VECTOR_SIZE_NAME "vec_size" #define H5D_XFER_HYPER_VECTOR_SIZE_SIZE sizeof(size_t) #define H5D_XFER_HYPER_VECTOR_SIZE_DEF 1024 +/* Definitions for EDC property */ +#define H5D_XFER_EDC_NAME "error-detecting" +#define H5D_XFER_EDC_SIZE sizeof(H5Z_EDC_t) +#define H5D_XFER_EDC_DEF H5Z_ENABLE_EDC +/* Definitions for filter callback function property */ +#define H5D_XFER_FILTER_CB_NAME "filter_cb" +#define H5D_XFER_FILTER_CB_SIZE sizeof(H5Z_cb_t) +#define H5D_XFER_FILTER_CB_DEF {NULL,NULL} /* * A dataset is the following struct. diff --git a/src/H5Fistore.c b/src/H5Fistore.c index aa88704..eefa690 100644 --- a/src/H5Fistore.c +++ b/src/H5Fistore.c @@ -898,12 +898,15 @@ H5F_istore_flush_entry(H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t reset) void *buf=NULL; /*temporary buffer */ size_t alloc; /*bytes allocated for BUF */ hbool_t point_of_no_return = FALSE; - + H5Z_cb_t cb_struct={NULL,NULL}; + H5Z_EDC_t edc=H5Z_ENABLE_EDC; + FUNC_ENTER_NOINIT(H5F_istore_flush_entry); assert(f); assert(ent); assert(!ent->locked); + HDmemset(&udata, 0, sizeof(H5F_istore_ud1_t)); buf = ent->chunk; if (ent->dirty) { @@ -941,8 +944,11 @@ H5F_istore_flush_entry(H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t reset) point_of_no_return = TRUE; ent->chunk = NULL; } - if (H5Z_pipeline(f, ent->pline, 0, &(udata.key.filter_mask), - &(udata.key.nbytes), &alloc, &buf)<0) { + /* Don't know whether we should involve transfer property list. So + * just pass in H5Z_ENABLE_EDC and default callback setting for data + * read. */ + if (H5Z_pipeline(f, ent->pline, 0, &(udata.key.filter_mask), edc, + cb_struct, &(udata.key.nbytes), &alloc, &buf)<0) { HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed"); } @@ -1322,13 +1328,19 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, H5F_istore_ud1_t udata; /*B-tree pass-through */ size_t chunk_size=0; /*size of a chunk */ hsize_t tempchunk_size; - size_t chunk_alloc=0; /*allocated chunk size */ herr_t status; /*func return status */ void *chunk=NULL; /*the file chunk */ void *ret_value; /*return value */ H5P_genplist_t *plist=NULL; /* Property list */ + H5Z_EDC_t edc; + H5Z_cb_t cb_struct; FUNC_ENTER_NOINIT(H5F_istore_lock); + + assert(TRUE==H5P_isa_class(dxpl_id,H5P_DATASET_XFER)); + plist=H5I_object(dxpl_id); + assert(plist!=NULL); + HDmemset(&udata, 0, sizeof(H5F_istore_ud1_t)); if (rdcc->nslots>0) { for (u=0, temp_idx=0; u<layout->ndims; u++) { @@ -1371,11 +1383,11 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, for (u=0, tempchunk_size=1; u<layout->ndims; u++) tempchunk_size *= layout->dim[u]; H5_ASSIGN_OVERFLOW(chunk_size,tempchunk_size,hsize_t,size_t); - chunk_alloc = chunk_size; - if (NULL==(chunk=H5MM_malloc (chunk_alloc))) + if (NULL==(chunk=H5MM_malloc (chunk_size))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); } else { + /* * Not in the cache. Read it from the file and count this as a miss * if it's in the file or an init if it isn't. @@ -1385,40 +1397,54 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, tempchunk_size *= layout->dim[u]; } H5_ASSIGN_OVERFLOW(chunk_size,tempchunk_size,hsize_t,size_t); - chunk_alloc = chunk_size; udata.mesg = *layout; udata.addr = HADDR_UNDEF; status = H5B_find (f, H5B_ISTORE, layout->addr, &udata); H5E_clear (); - if (NULL==(chunk = H5MM_malloc (chunk_alloc))) - HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); + if (status>=0 && H5F_addr_defined(udata.addr)) { + size_t chunk_alloc=0; /*allocated chunk size */ + /* * The chunk exists on disk. */ + /* Chunk size on disk isn't [likely] the same size as the final chunk + * size in memory, so allocate memory big enough. */ + chunk_alloc = udata.key.nbytes; + if (NULL==(chunk = H5MM_malloc (chunk_alloc))) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); if (H5F_block_read(f, H5FD_MEM_DRAW, udata.addr, udata.key.nbytes, H5P_DATASET_XFER_DEFAULT, chunk)<0) HGOTO_ERROR (H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk"); - if (H5Z_pipeline(f, pline, H5Z_FLAG_REVERSE, - &(udata.key.filter_mask), &(udata.key.nbytes), - &chunk_alloc, &chunk)<0 || udata.key.nbytes!=chunk_size) + if(H5P_get(plist,H5D_XFER_EDC_NAME,&edc)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get edc information"); + if(H5P_get(plist,H5D_XFER_FILTER_CB_NAME,&cb_struct)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get filter callback struct"); + if (H5Z_pipeline(f, pline, H5Z_FLAG_REVERSE, &(udata.key.filter_mask), edc, + cb_struct, &(udata.key.nbytes), &chunk_alloc, &chunk)<0) { HGOTO_ERROR(H5E_PLINE, H5E_READERROR, NULL, "data pipeline read failed"); + } rdcc->nmisses++; - } else if (fill && fill->buf) { - /* - * The chunk doesn't exist in the file. Replicate the fill - * value throughout the chunk. - */ - assert(0==chunk_size % fill->size); - H5V_array_fill(chunk, fill->buf, fill->size, chunk_size/fill->size); - rdcc->ninits++; } else { - /* - * The chunk doesn't exist in the file and no fill value was - * specified. Assume all zeros. - */ - HDmemset (chunk, 0, chunk_size); + /* Chunk size on disk isn't [likely] the same size as the final chunk + * size in memory, so allocate memory big enough. */ + if (NULL==(chunk = H5MM_malloc (chunk_size))) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk"); + if (fill && fill->buf) { + /* + * The chunk doesn't exist in the file. Replicate the fill + * value throughout the chunk. + */ + assert(0==chunk_size % fill->size); + H5V_array_fill(chunk, fill->buf, fill->size, chunk_size/fill->size); + } else { + /* + * The chunk doesn't exist in the file and no fill value was + * specified. Assume all zeros. + */ + HDmemset (chunk, 0, chunk_size); + } rdcc->ninits++; - } + } /* end else */ } assert (found || chunk_size>0); @@ -1436,6 +1462,7 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, if (H5F_istore_preempt(f, ent, TRUE)<0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache"); } + if (H5F_istore_prune(f, chunk_size)<0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache"); @@ -1453,9 +1480,6 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, ent->wr_count = chunk_size; ent->chunk = chunk; - assert(TRUE==H5P_isa_class(dxpl_id,H5P_DATASET_XFER)); - plist=H5I_object(dxpl_id); - assert(plist!=NULL); H5P_get(plist,H5D_XFER_BTREE_SPLIT_RATIO_NAME,&(ent->split_ratios)); /* Add it to the cache */ @@ -1476,7 +1500,6 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, ent->prev = NULL; } found = TRUE; - } else if (!found) { /* * The chunk is larger than the entire cache so we don't cache it. @@ -1523,7 +1546,8 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, done: if (!ret_value) - H5MM_xfree (chunk); + if(chunk) + H5MM_xfree (chunk); FUNC_LEAVE_NOAPI(ret_value); } @@ -2337,6 +2361,8 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, unsigned chunk_exists; /* Flag to indicate whether a chunk exists already */ int i; /* Local index variable */ unsigned u; /* Local index variable */ + H5Z_EDC_t edc; /* Decide whether to enable EDC for read */ + H5Z_cb_t cb_struct; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5F_istore_allocate, FAIL); @@ -2363,6 +2389,10 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset transfer property list"); if(H5P_get(dx_plist,H5D_XFER_BTREE_SPLIT_RATIO_NAME,split_ratios)<0) HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "can't get B-tree split ratios"); + if(H5P_get(dx_plist,H5D_XFER_EDC_NAME,&edc)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get edc information"); + if(H5P_get(dx_plist,H5D_XFER_FILTER_CB_NAME,&cb_struct)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get filter callback struct"); #ifdef H5_HAVE_PARALLEL /* Retrieve up MPI parameters */ @@ -2435,7 +2465,7 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, size_t nbytes=(size_t)chunk_size; /* Push the chunk through the filters */ - if (H5Z_pipeline(f, &pline, 0, &filter_mask, &nbytes, &buf_size, &chunk)<0) + if (H5Z_pipeline(f, &pline, 0, &filter_mask, edc, cb_struct, &nbytes, &buf_size, &chunk)<0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed"); /* Keep the number of bytes the chunk turned in to */ diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 7aafe92..e299c56 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -800,6 +800,48 @@ done: FUNC_LEAVE_API(ret_value); } + +/*------------------------------------------------------------------------- + * Function: H5Pset_adler32 + * + * Purpose: Sets Adler32 checksum of EDC for a dataset creation + * property list. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * Dec 19, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_adler32(hid_t plist_id) +{ + H5O_pline_t pline; + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value=SUCCEED; /* return value */ + + FUNC_ENTER_API(H5Pset_adler32, FAIL); + H5TRACE1("e","i",plist_id); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + + /* Add the Adler32 checksum as a filter */ + if(H5P_get(plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline"); + if(H5Z_append(&pline, H5Z_FILTER_ADLER32, H5Z_FLAG_MANDATORY, 0, NULL)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to add deflate filter to pipeline"); + if(H5P_set(plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to set pipeline"); + +done: + FUNC_LEAVE_API(ret_value); +} + /*------------------------------------------------------------------------- * Function: H5Pset_fill_value diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 33aa607..2f7bb82 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -421,6 +421,131 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pset_edc_check + * + * Purpose: Enable or disable error-detecting for a dataset reading + * process. This error-detecting algorithm is whichever + * user chooses earlier. This function cannot control + * writing process. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * Jan 3, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_edc_check(hid_t plist_id, H5Z_EDC_t check) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value=SUCCEED; /* return value */ + + FUNC_ENTER_API(H5Pset_edc_check, FAIL); + + /* Check argument */ + if (check != H5Z_ENABLE_EDC && check != H5Z_DISABLE_EDC) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid value"); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + + /* Update property list */ + if (H5P_set(plist,H5D_XFER_EDC_NAME,&check)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value"); + +done: + FUNC_LEAVE_API(ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_edc_check + * + * Purpose: Enable or disable error-detecting for a dataset reading + * process. This error-detecting algorithm is whichever + * user chooses earlier. This function cannot control + * writing process. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * Jan 3, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +H5Z_EDC_t +H5Pget_edc_check(hid_t plist_id) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5Z_EDC_t ret_value; /* return value */ + + FUNC_ENTER_API(H5Pget_edc_check, FAIL); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + + /* Update property list */ + if (H5P_get(plist,H5D_XFER_EDC_NAME,&ret_value)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value"); + + /* check valid value */ + if (ret_value != H5Z_ENABLE_EDC && ret_value != H5Z_DISABLE_EDC) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid value"); + +done: + FUNC_LEAVE_API(ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pset_filter_callback + * + * Purpose: Sets user's callback function for dataset transfer property + * list. This callback function defines what user wants to do + * if certain filter fails. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * Jan 14, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_filter_callback(hid_t plist_id, H5Z_filter_func_t func, void* op_data) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value=SUCCEED; /* return value */ + H5Z_cb_t cb_struct; + + FUNC_ENTER_API(H5Pset_filter_callback, FAIL); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_XFER))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + + /* Update property list */ + cb_struct.func = func; + cb_struct.op_data = op_data; + + if (H5P_set(plist,H5D_XFER_FILTER_CB_NAME,&cb_struct)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value"); + +done: + FUNC_LEAVE_API(ret_value); +} + + +/*------------------------------------------------------------------------- * Function: H5Pget_btree_ratios * * Purpose: Queries B-tree split ratios. See H5Pset_btree_ratios(). diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index abe8e63..fbe1aea 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -219,6 +219,11 @@ H5_DLL H5Z_filter_t H5Pget_filter(hid_t plist_id, int filter, size_t namelen, char name[]); H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned aggression); H5_DLL herr_t H5Pset_shuffle(hid_t plist_id, unsigned bytespertype); +H5_DLL herr_t H5Pset_adler32(hid_t plist_id); +H5_DLL herr_t H5Pset_edc_check(hid_t plist_id, H5Z_EDC_t check); +H5_DLL H5Z_EDC_t H5Pget_edc_check(hid_t plist_id); +H5_DLL herr_t H5Pset_filter_callback(hid_t plist_id, H5Z_filter_func_t func, + void* op_data); #ifdef H5_WANT_H5_V1_4_COMPAT H5_DLL herr_t H5Pset_cache(hid_t plist_id, int mdc_nelmts, int rdcc_nelmts, size_t rdcc_nbytes, double rdcc_w0); diff --git a/src/H5Tconv.c b/src/H5Tconv.c index b855d88..a6f8dbb 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -2239,7 +2239,7 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, hsize_t nelmts, /* Get length of element sequences */ if((seq_len=(*(src->u.vlen.getlen))(src->u.vlen.f,s))<0) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "null pointer"); + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "incorrect length"); H5_CHECK_OVERFLOW(seq_len,hssize_t,size_t); src_size=(size_t)seq_len*src_base_size; dst_size=(size_t)seq_len*dst_base_size; diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c index 42e249a..fa65056 100644 --- a/src/H5Tvlen.c +++ b/src/H5Tvlen.c @@ -291,7 +291,10 @@ H5T_vlen_str_mem_getlen(H5F_t UNUSED *f, void *vl_addr) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "null pointer"); /* Set return value */ - ret_value=(hssize_t)HDstrlen(s); + if(s) + ret_value=(hssize_t)HDstrlen(s); + else + ret_value = 0; done: FUNC_LEAVE_NOAPI(ret_value); @@ -324,7 +327,10 @@ H5T_vlen_str_mem_read(H5F_t UNUSED *f, void *vl_addr, void *buf, size_t len) assert(s); assert(buf); - HDmemcpy(buf,s,len); + if(s && buf && len>0) + HDmemcpy(buf,s,len); + if(!s && len==-1) + buf = NULL; done: FUNC_LEAVE_NOAPI(ret_value); @@ -52,6 +52,9 @@ H5Z_init_interface (void) #ifdef H5_HAVE_FILTER_SHUFFLE H5Z_register (H5Z_FILTER_SHUFFLE, "shuffle", H5Z_filter_shuffle); #endif /* H5_HAVE_FILTER_SHUFFLE */ +#ifdef H5_HAVE_FILTER_ADLER32 + H5Z_register (H5Z_FILTER_ADLER32, "adler32", H5Z_filter_adler32); +#endif /* H5_HAVE_FILTER_ADLER32 */ FUNC_LEAVE_NOAPI(SUCCEED); } @@ -492,12 +495,14 @@ done: */ herr_t H5Z_pipeline(H5F_t UNUSED *f, const H5O_pline_t *pline, unsigned flags, - unsigned *filter_mask/*in,out*/, size_t *nbytes/*in,out*/, - size_t *buf_size/*in,out*/, void **buf/*in,out*/) + unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, + H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, + size_t *buf_size/*in,out*/, void **buf/*in,out*/) { size_t i, idx, new_nbytes; H5Z_class_t *fclass=NULL; unsigned failed = 0; + unsigned tmp_flags; #ifdef H5Z_DEBUG H5_timer_t timer; #endif @@ -513,7 +518,7 @@ H5Z_pipeline(H5F_t UNUSED *f, const H5O_pline_t *pline, unsigned flags, assert(buf && *buf); assert(!pline || pline->nfilters<32); - if (pline && (flags & H5Z_FLAG_REVERSE)) { + if (pline && (flags & H5Z_FLAG_REVERSE)) { /* Read */ for (i=pline->nfilters; i>0; --i) { idx = i-1; @@ -528,22 +533,30 @@ H5Z_pipeline(H5F_t UNUSED *f, const H5O_pline_t *pline, unsigned flags, #ifdef H5Z_DEBUG H5_timer_begin(&timer); #endif - new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags), - pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, - *nbytes, buf_size, buf); + tmp_flags=flags|(pline->filter[idx].flags); + tmp_flags|=(edc_read== H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; + new_nbytes = (fclass->func)(tmp_flags, pline->filter[idx].cd_nelmts, + pline->filter[idx].cd_values, *nbytes, buf_size, buf); + #ifdef H5Z_DEBUG H5_timer_end(&(fclass->stats[1].timer), &timer); fclass->stats[1].total += MAX(*nbytes, new_nbytes); if (0==new_nbytes) fclass->stats[1].errors += *nbytes; #endif - if (0==new_nbytes) { - failed |= (unsigned)1 << idx; - HGOTO_ERROR(H5E_PLINE, H5E_READERROR, FAIL, "filter returned failure"); - } - *nbytes = new_nbytes; + + if(0==new_nbytes) { + if((cb_struct.func && (H5Z_CB_FAIL==cb_struct.func(pline->filter[idx].id, *buf, *buf_size, cb_struct.op_data))) + || !cb_struct.func) { + failed |= (unsigned)1 << idx; + HGOTO_ERROR(H5E_PLINE, H5E_READERROR, FAIL, "filter returned failure during read"); + } else { + H5E_clear(); + *nbytes = *buf_size; + } + } else + *nbytes = new_nbytes; } - } else if (pline) { + } else if (pline) { /* Write */ for (idx=0; idx<pline->nfilters; idx++) { if (*filter_mask & ((unsigned)1<<idx)) { failed |= (unsigned)1 << idx; @@ -561,25 +574,29 @@ H5Z_pipeline(H5F_t UNUSED *f, const H5O_pline_t *pline, unsigned flags, #ifdef H5Z_DEBUG H5_timer_begin(&timer); #endif - new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags), - pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, - *nbytes, buf_size, buf); + new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags), pline->filter[idx].cd_nelmts, + pline->filter[idx].cd_values, *nbytes, buf_size, buf); #ifdef H5Z_DEBUG H5_timer_end(&(fclass->stats[0].timer), &timer); fclass->stats[0].total += MAX(*nbytes, new_nbytes); if (0==new_nbytes) fclass->stats[0].errors += *nbytes; #endif - if (0==new_nbytes) { - failed |= (unsigned)1 << idx; - if (0==(pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) { - HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "filter returned failure"); - } else { - H5E_clear(); - } - } else { - *nbytes = new_nbytes; - } + if(0==new_nbytes) { + if (0==(pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) { + if((cb_struct.func && (H5Z_CB_FAIL==cb_struct.func(pline->filter[idx].id, *buf, *nbytes, cb_struct.op_data))) + || !cb_struct.func) { + failed |= (unsigned)1 << idx; + HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "filter returned failure"); + } else { + H5E_clear(); + *nbytes = *buf_size; + } + } else { + H5E_clear(); + } + } else { + *nbytes = new_nbytes; + } } } diff --git a/src/H5Zadler32.c b/src/H5Zadler32.c new file mode 100644 index 0000000..53ce12d --- /dev/null +++ b/src/H5Zadler32.c @@ -0,0 +1,139 @@ +/* + * Copyright © 1999-2001 NCSA + * All rights reserved. + * + * Programmer: Raymond Lu<slu@ncsa.uiuc.edu> + * Jan 3, 2003 + */ +#include "H5private.h" +#include "H5Eprivate.h" +#include "H5MMprivate.h" +#include "H5Zprivate.h" + +#ifdef H5_HAVE_FILTER_ADLER32 + +#define ADLER_LEN 4 +#define ADLER_BASE 65521 + +/* Interface initialization */ +#define PABLO_MASK H5Z_adler32_mask +#define INTERFACE_INIT NULL +static int interface_initialize_g = 0; + + +/*------------------------------------------------------------------------- + * Function: H5Z_filter_adler32_compute + * + * Purpose: Implement an Adler32 Checksum + * + * Return: Success: Adler32 value + * + * Failure: Can't fail + * + * Programmer: Raymond Lu + * Jan 3, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static unsigned int H5Z_filter_adler32_compute(unsigned char *buf, size_t len) +{ + size_t i; + register unsigned int s1 = 1; + register unsigned int s2 = 0; + + FUNC_ENTER_NOINIT(H5Z_filter_adler32_compute); + + /* Compute checksum */ + for(i=0; i<len; i++) { + s1 = (s1 + *buf++) % ADLER_BASE; + s2 = (s2 + s1) % ADLER_BASE; + } + + FUNC_LEAVE_NOAPI((s2 << 16) + s1); +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_filter_adler32 + * + * Purpose: Implement an I/O filter of Adler32 Checksum + * + * Return: Success: size of data plus the size of Adler32 value + * + * Failure: 0 + * + * Programmer: Raymond Lu + * Jan 3, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5Z_filter_adler32 (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +{ + size_t ret_value = 0; + void *outbuf = NULL; + + unsigned char *src = (unsigned char*)(*buf); + unsigned int adler = 1; + + FUNC_ENTER_NOAPI(H5Z_filter_adler32, 0); + + assert(sizeof(unsigned int)==4); + + if (flags & H5Z_FLAG_REVERSE) { /* Read */ + size_t src_nbytes = nbytes; + unsigned int origin_adler; + + /* Do checksum if it's enabled for read; otherwise skip it + * to save performance. */ + if (!(flags & H5Z_FLAG_SKIP_EDC)) { /* Read */ + unsigned char *tmp_src; + + src_nbytes -= ADLER_LEN; + tmp_src=src+src_nbytes; + UINT32DECODE(tmp_src, origin_adler); + + /* Compute checksum */ + adler = H5Z_filter_adler32_compute(src,src_nbytes); + + if(origin_adler != adler) + HGOTO_ERROR(H5E_STORAGE, H5E_READERROR, 0, "data error detected by Adler32 checksum"); + } + + *buf_size = nbytes - ADLER_LEN; + ret_value = *buf_size; + } else { /* Write */ + unsigned char *dst; + + /* Compute checksum */ + adler = H5Z_filter_adler32_compute(src,nbytes); + + if (NULL==(dst=outbuf=H5MM_malloc(nbytes+ADLER_LEN))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate Adler32 checksum destination buffer"); + + /* Copy raw data */ + HDmemcpy((void*)dst, (void*)(*buf), nbytes); + + /* Append checksum to raw data */ + dst += nbytes; + UINT32ENCODE(dst, adler); + + *buf_size = nbytes + ADLER_LEN; + H5MM_xfree(*buf); + *buf = outbuf; + outbuf = NULL; + ret_value = *buf_size; + } + +done: + if(outbuf) + H5MM_xfree(outbuf); + FUNC_LEAVE_NOAPI(ret_value); +} + +#endif /* H5_HAVE_FILTER_ADLER32 */ diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index 3c91836..c147533 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -21,6 +21,8 @@ #define INTERFACE_INIT NULL static int interface_initialize_g = 0; +#define H5Z_DEFLATE_SIZE_ADJUST(s) (HDceil((double)((s)*1.001))+12) + /*------------------------------------------------------------------------- * Function: H5Z_filter_deflate @@ -103,10 +105,10 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, */ const Bytef *z_src = (const Bytef*)(*buf); Bytef *z_dst; /*destination buffer */ - uLongf z_dst_nbytes = (uLongf)nbytes; + uLongf z_dst_nbytes = (uLongf)H5Z_DEFLATE_SIZE_ADJUST(nbytes); uLong z_src_nbytes = (uLong)nbytes; - - if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes))) + + if (NULL==(z_dst=outbuf=H5MM_malloc(z_dst_nbytes))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate deflate destination buffer"); status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); if (Z_BUF_ERROR==status) { @@ -114,7 +116,7 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, } else if (Z_MEM_ERROR==status) { HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, 0, "deflate memory error"); } else if (Z_OK!=status) { - HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, 0, "deflate error"); + HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, 0, "other deflate error"); } else { H5MM_xfree(*buf); *buf = outbuf; diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h index 5f05dc8..b150d0f 100644 --- a/src/H5Zprivate.h +++ b/src/H5Zprivate.h @@ -10,6 +10,7 @@ #include "H5Zpublic.h" #include "H5Fprivate.h" +#include "H5Ppublic.h" /* * The filter table maps filter identification numbers to structs that @@ -39,8 +40,9 @@ H5_DLL herr_t H5Z_append(struct H5O_pline_t *pline, H5Z_filter_t filter, const unsigned int cd_values[]); H5_DLL herr_t H5Z_pipeline(H5F_t *f, const struct H5O_pline_t *pline, unsigned flags, unsigned *filter_mask/*in,out*/, - size_t *nbytes/*in,out*/, - size_t *buf_size/*in,out*/, void **buf/*in,out*/); + H5Z_EDC_t edc_read, H5Z_cb_t cb_struct, + size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, + void **buf/*in,out*/); H5_DLL H5Z_class_t *H5Z_find(H5Z_filter_t id); @@ -52,4 +54,9 @@ H5_DLL size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts, H5_DLL size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +H5_DLL size_t H5Z_filter_adler32(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + #endif diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 67b1fe9..f45c96c 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -19,17 +19,46 @@ typedef int H5Z_filter_t; #define H5Z_FILTER_ERROR (-1) /*no filter */ #define H5Z_FILTER_NONE 0 /*reserved indefinitely */ #define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */ -#define H5Z_FILTER_SHUFFLE 2 /* shuffle the data */ +#define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */ +#define H5Z_FILTER_ADLER32 3 /*adler32 checksum of EDC */ #define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved */ #define H5Z_FILTER_MAX 65535 /*maximum filter id */ /* Flags for filter definition */ #define H5Z_FLAG_DEFMASK 0x00ff /*definition flag mask */ +#define H5Z_FLAG_MANDATORY 0x0000 /*filter is mandatory */ #define H5Z_FLAG_OPTIONAL 0x0001 /*filter is optional */ /* Additional flags for filter invocation */ #define H5Z_FLAG_INVMASK 0xff00 /*invocation flag mask */ #define H5Z_FLAG_REVERSE 0x0100 /*reverse direction; read */ +#define H5Z_FLAG_SKIP_EDC 0x0200 /*skip EDC filters for read */ + +/* Values to decide if EDC is enabled for reading data */ +typedef enum H5Z_EDC_t { + H5Z_ERROR_EDC = -1, /* error value */ + H5Z_DISABLE_EDC = 0, + H5Z_ENABLE_EDC = 1, + H5Z_NO_EDC = 2 /* must be the last */ +} H5Z_EDC_t; + +/* Return values for filter callback function */ +typedef enum H5Z_cb_return_t { + H5Z_CB_ERROR = -1, + H5Z_CB_FAIL = 0, /* I/O should fail if filter fails. */ + H5Z_CB_CONT = 1, /* I/O continues if filter fails. */ + H5Z_CB_NO = 2 +} H5Z_cb_return_t; + +/* Filter callback function definition */ +typedef H5Z_cb_return_t (*H5Z_filter_func_t)(H5Z_filter_t filter, void* buf, + size_t buf_size, void* op_data); + +/* Structure for filter callback property */ +typedef struct H5Z_cb_t { + H5Z_filter_func_t func; + void* op_data; +} H5Z_cb_t; #ifdef __cplusplus extern "C" { diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index f25b5ff..a33d3cd 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -42,9 +42,8 @@ static int interface_initialize_g = 0; *------------------------------------------------------------------------- */ size_t -H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf) +H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf) { void *dest = NULL; /* Buffer to deposit [un]shuffled bytes into */ unsigned char *_src; /* Alias for source buffer */ @@ -52,6 +51,7 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, unsigned bytesoftype; /* Number of bytes per element */ size_t numofelements; /* Number of elements in buffer */ size_t i,j; /* Local index variables */ + size_t leftover; /* Extra bytes at end of buffer */ size_t ret_value; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_shuffle, 0); @@ -68,6 +68,9 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, /* Compute the number of elements in buffer */ numofelements=nbytes/bytesoftype; + /* Compute the leftover bytes if there are any */ + leftover = nbytes%bytesoftype; + /* Allocate the destination buffer */ if (NULL==(dest = H5MM_malloc(nbytes))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for shuffle buffer"); @@ -84,6 +87,13 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, _dest+=bytesoftype; } /* end for */ } /* end for */ + + /* Add leftover to the end of data */ + if(leftover>0) { + /* Adjust back to end of shuffled bytes */ + _dest -= (bytesoftype - 1); + HDmemcpy((void*)_dest, (void*)_src, leftover); + } } /* end if */ else { /* Get the pointer to the destination buffer */ @@ -97,6 +107,13 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, _src+=bytesoftype; } /* end for */ } /* end for */ + + /* Add leftover to the end of data */ + if(leftover>0) { + /* Adjust back to end of shuffled bytes */ + _src -= (bytesoftype - 1); + HDmemcpy((void*)_dest, (void*)_src, leftover); + } } /* end else */ /* Set the buffer information to return */ diff --git a/src/H5config.h.in b/src/H5config.h.in index 09d84e7..1b7f58c 100644 --- a/src/H5config.h.in +++ b/src/H5config.h.in @@ -27,6 +27,9 @@ /* Define if support for shuffle filter is enabled */ #undef HAVE_FILTER_SHUFFLE +/* Define if support for Adler32 checksum filter is enabled */ +#undef HAVE_FILTER_ADLER32 + /* Define to 1 if you have the `fork' function. */ #undef HAVE_FORK diff --git a/src/Makefile.in b/src/Makefile.in index 95a0fc5..6b28099 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -39,7 +39,7 @@ LIB_SRC=H5.c H5A.c H5AC.c H5B.c H5D.c H5E.c H5F.c H5Farray.c H5Fcontig.c \ H5P.c H5Pdcpl.c H5Pdxpl.c H5Pfapl.c H5Pfcpl.c H5R.c H5RS.c H5S.c \ H5Sall.c H5Shyper.c H5Smpio.c H5Snone.c H5Spoint.c H5Sselect.c H5ST.c \ H5T.c H5Tbit.c H5Tconv.c H5Tinit.c H5Tvlen.c H5TB.c H5TS.c H5V.c \ - H5Z.c H5Zdeflate.c H5Zshuffle.c + H5Z.c H5Zdeflate.c H5Zshuffle.c H5Zadler32.c LIB_OBJ=$(LIB_SRC:.c=.lo) |