diff options
author | Mohamad Chaarawi <chaarawi@hdfgroup.org> | 2013-10-31 18:36:50 (GMT) |
---|---|---|
committer | Mohamad Chaarawi <chaarawi@hdfgroup.org> | 2013-10-31 18:36:50 (GMT) |
commit | c2641d2d1c1ee2db8c4cfb472c8f6f2b5edd1680 (patch) | |
tree | 636dc8559d50f7d3300513a63fd7aaa95fe1902f /src | |
parent | 49ed87466d33047abff0276b319837276ee2a35b (diff) | |
download | hdf5-c2641d2d1c1ee2db8c4cfb472c8f6f2b5edd1680.zip hdf5-c2641d2d1c1ee2db8c4cfb472c8f6f2b5edd1680.tar.gz hdf5-c2641d2d1c1ee2db8c4cfb472c8f6f2b5edd1680.tar.bz2 |
[svn-r24388] switch to using Mercury 64 bit CRC algorithm instead of internal checksum algorithm.
Diffstat (limited to 'src')
-rw-r--r-- | src/H5Pdxpl.c | 12 | ||||
-rw-r--r-- | src/H5Pencdec.c | 70 | ||||
-rw-r--r-- | src/H5Ppkg.h | 2 | ||||
-rw-r--r-- | src/H5Sprivate.h | 2 | ||||
-rw-r--r-- | src/H5Sselect.c | 28 | ||||
-rw-r--r-- | src/H5VLiod.c | 72 | ||||
-rw-r--r-- | src/H5VLiod.h | 19 | ||||
-rw-r--r-- | src/H5VLiod_analysis.c | 2 | ||||
-rw-r--r-- | src/H5VLiod_attr.c | 6 | ||||
-rw-r--r-- | src/H5VLiod_client.c | 73 | ||||
-rw-r--r-- | src/H5VLiod_client.h | 8 | ||||
-rw-r--r-- | src/H5VLiod_common.h | 12 | ||||
-rw-r--r-- | src/H5VLiod_dset.c | 62 | ||||
-rw-r--r-- | src/H5VLiod_dtype.c | 6 | ||||
-rw-r--r-- | src/H5VLiod_file.c | 2 | ||||
-rw-r--r-- | src/H5VLiod_group.c | 2 | ||||
-rw-r--r-- | src/H5VLiod_map.c | 14 | ||||
-rw-r--r-- | src/H5VLiod_obj.c | 4 | ||||
-rw-r--r-- | src/H5VLiod_server.h | 2 | ||||
-rw-r--r-- | src/H5VLiod_util.c | 8 | ||||
-rw-r--r-- | src/H5checksum.c | 60 | ||||
-rw-r--r-- | src/H5public.h | 3 |
22 files changed, 320 insertions, 149 deletions
diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 536b6be..578e8dd 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -55,10 +55,10 @@ #define H5D_XFER_INJECT_CORRUPTION_ENC H5P__encode_hbool_t #define H5D_XFER_INJECT_CORRUPTION_DEC H5P__decode_hbool_t -#define H5D_XFER_CHECKSUM_SIZE sizeof(uint32_t) +#define H5D_XFER_CHECKSUM_SIZE sizeof(uint64_t) #define H5D_XFER_CHECKSUM_DEF 0 -#define H5D_XFER_CHECKSUM_ENC H5P__encode_unsigned -#define H5D_XFER_CHECKSUM_DEC H5P__decode_unsigned +#define H5D_XFER_CHECKSUM_ENC H5P__encode_uint64_t +#define H5D_XFER_CHECKSUM_DEC H5P__decode_uint64_t /* definitions for checksum scope in FF stack */ #define H5D_XFER_CHECKSUM_SCOPE_SIZE sizeof(uint32_t) @@ -66,7 +66,7 @@ #define H5D_XFER_CHECKSUM_SCOPE_ENC H5P__encode_unsigned #define H5D_XFER_CHECKSUM_SCOPE_DEC H5P__decode_unsigned -#define H5D_XFER_CHECKSUM_PTR_SIZE sizeof(uint32_t *) +#define H5D_XFER_CHECKSUM_PTR_SIZE sizeof(uint64_t *) #define H5D_XFER_CHECKSUM_PTR_DEF NULL #endif /* H5_HAVE_EFF */ @@ -263,8 +263,8 @@ const H5P_libclass_t H5P_CLS_DXFR[1] = {{ #ifdef H5_HAVE_EFF static const hbool_t H5D_def_inject_corruption_g = H5D_XFER_INJECT_CORRUPTION_DEF; -static const uint32_t H5D_def_checksum_g = H5D_XFER_CHECKSUM_DEF; -static const uint32_t *H5D_def_checksum_ptr_g = H5D_XFER_CHECKSUM_PTR_DEF; +static const uint64_t H5D_def_checksum_g = H5D_XFER_CHECKSUM_DEF; +static const uint64_t *H5D_def_checksum_ptr_g = H5D_XFER_CHECKSUM_PTR_DEF; static const uint32_t H5D_def_checksum_scope_g = H5D_XFER_CHECKSUM_SCOPE_DEF; #endif /* H5_HAVE_EFF */ diff --git a/src/H5Pencdec.c b/src/H5Pencdec.c index 4df23ab..149108e 100644 --- a/src/H5Pencdec.c +++ b/src/H5Pencdec.c @@ -184,6 +184,42 @@ H5P__encode_hsize_t(const void *value, void **_pp, size_t *size) /*------------------------------------------------------------------------- + * Function: H5P__encode_uint64_t + * + * Purpose: Generic encoding callback routine for 'uint64_t' properties. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Mohamad Chaarawi + * August 07, 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5P__encode_uint64_t(const void *value, void **_pp, size_t *size) +{ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(value); + HDassert(size); + + if(NULL != *pp) { + /* Encode the value */ + UINT64ENCODE(*pp, *(const uint64_t *)value); + } /* end if */ + + /* Set size needed for encoding */ + *size += sizeof(uint64_t); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__encode_uint64_t() */ + + +/*------------------------------------------------------------------------- * Function: H5P__encode_unsigned * * Purpose: Generic encoding callback routine for 'unsigned' properties. @@ -551,6 +587,40 @@ H5P__decode_hsize_t(const void **_pp, void *_value) /*------------------------------------------------------------------------- + * Function: H5P__decode_uint64_t + * + * Purpose: Generic encoding callback routine for 'uint64_t' properties. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Mohamad Chaarawi + * August 07, 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5P__decode_uint64_t(const void **_pp, void *_value) +{ + uint64_t *value = (uint64_t *)_value; /* Property value to return */ + const uint8_t **pp = (const uint8_t **)_pp; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(value); + + UINT64DECODE(*pp, *value) + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__decode_uint64_t() */ + + +/*------------------------------------------------------------------------- * Function: H5P__decode_unsigned * * Purpose: Generic decoding callback routine for 'unsigned' properties. diff --git a/src/H5Ppkg.h b/src/H5Ppkg.h index 9d48c60..3073dd7 100644 --- a/src/H5Ppkg.h +++ b/src/H5Ppkg.h @@ -211,12 +211,14 @@ H5_DLL herr_t H5P__encode_hsize_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_size_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_unsigned(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_uint8_t(const void *value, void **_pp, size_t *size); +H5_DLL herr_t H5P__encode_uint64_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_hbool_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__encode_double(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__decode_hsize_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_size_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_unsigned(const void **_pp, void *value); H5_DLL herr_t H5P__decode_uint8_t(const void **_pp, void *value); +H5_DLL herr_t H5P__decode_uint64_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_hbool_t(const void **_pp, void *value); H5_DLL herr_t H5P__decode_double(const void **_pp, void *value); diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 34fc07f..bee64a2 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -272,7 +272,7 @@ H5_DLL H5S_t *H5S_decode(const unsigned char *buf); H5_DLL herr_t H5S_get_offsets(const H5S_t *space, size_t elmt_size, size_t nelmts, hsize_t **_off, size_t **_len, size_t *_num_entries); -H5_DLL uint32_t H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, +H5_DLL uint64_t H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, const H5S_t *space); #ifdef H5_HAVE_PARALLEL diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 6e45e23..94d87fb 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -31,6 +31,7 @@ #include "H5Spkg.h" /* Dataspaces */ #include "H5Vprivate.h" /* Vector and array functions */ #include "H5WBprivate.h" /* Wrapped Buffers */ +#include "mchecksum.h" /* Mercury Checksum library */ /* Local functions */ #ifdef LATER @@ -2160,7 +2161,7 @@ done: * *------------------------------------------------------------------------- */ -uint32_t +uint64_t H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, const H5S_t *space) { hsize_t _off[H5D_IO_VECTOR_SIZE]; /* Array to store sequence offsets in memory */ @@ -2170,7 +2171,7 @@ H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, const H5S_t *spac H5S_sel_iter_t iter; /* Memory selection iteration info */ hbool_t iter_init = 0; /* Memory selection iteration info has been initialized */ size_t nseq; /* Number of sequences generated */ - uint32_t ret_value = 0; + uint64_t ret_value = 0; FUNC_ENTER_NOAPI_NOINIT @@ -2184,13 +2185,15 @@ H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, const H5S_t *spac if(H5S_SELECT_OFFSET(space, off) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, 0, "can't retrieve memory selection offset"); - ret_value = H5_checksum_lookup4(buf, buf_size, NULL); + ret_value = H5_checksum_crc64(buf, buf_size); } /* end if */ else { size_t nelem; /* Number of elements used in sequences */ size_t i; const uint8_t *p = (const uint8_t *)buf; - H5_checksum_seed_t cs; + const char *hash_method = "crc64"; + size_t hash_size; + mchecksum_object_t checksum; /* Initialize iterator */ if(H5S_select_iter_init(&iter, space, elmt_size) < 0) @@ -2199,8 +2202,8 @@ H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, const H5S_t *spac nseq = 0; - cs.a = cs.b = cs.c = cs.state = 0; - //cs.total_length = elmt_size * nelmts; + /* Initialize checksum */ + mchecksum_init(hash_method, &checksum); /* Loop, until all bytes are processed */ while(nelmts > 0) { @@ -2210,10 +2213,21 @@ H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, const H5S_t *spac HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); for(i=0 ; i<nseq ; i++) { - ret_value = H5_checksum_lookup4(p+off[i], len[i], &cs); + mchecksum_update(checksum, p+off[i], len[i]); } nelmts -= nelem; } /* end while */ + + /* Get size of checksum */ + hash_size = mchecksum_get_size(checksum); + + assert(hash_size == sizeof(uint64_t)); + + /* get checksum value */ + mchecksum_get(checksum, &ret_value, hash_size, 1); + + /* Destroy checksum */ + mchecksum_destroy(checksum); } /* end else */ done: diff --git a/src/H5VLiod.c b/src/H5VLiod.c index 14e1ec4..d9f1aef 100644 --- a/src/H5VLiod.c +++ b/src/H5VLiod.c @@ -602,7 +602,7 @@ EFF_init(MPI_Comm comm, MPI_Info UNUSED info) using mercury */ /* Only rank 0 reads file */ if (my_rank == 0) { - int count, index=0, num_ions; + int count, line=0, num_ions; FILE *config; char config_addr_name[H5VL_IOD_MAX_ADDR_NAME]; @@ -615,22 +615,22 @@ EFF_init(MPI_Comm comm, MPI_Info UNUSED info) if(fgets(config_addr_name, H5VL_IOD_MAX_ADDR_NAME, config) != NULL) { strncpy(addr_name, config_addr_name, H5VL_IOD_MAX_ADDR_NAME); count = 1; - while(num_procs > index + (count*num_ions)) { + while(num_procs > line + (count*num_ions)) { MPI_Send(config_addr_name, H5VL_IOD_MAX_ADDR_NAME, MPI_BYTE, - index + (count*num_ions), tag, comm); + line + (count*num_ions), tag, comm); count ++; } - index++; + line++; } while (fgets(config_addr_name, H5VL_IOD_MAX_ADDR_NAME, config) != NULL) { count = 0; - while(num_procs > index + (count*num_ions)) { + while(num_procs > line + (count*num_ions)) { MPI_Send(config_addr_name, H5VL_IOD_MAX_ADDR_NAME, MPI_BYTE, - index + (count*num_ions), tag, comm); + line + (count*num_ions), tag, comm); count ++; } - index ++; + line ++; } fclose(config); } @@ -957,7 +957,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pset_dxpl_checksum(hid_t dxpl_id, uint32_t cs) +H5Pset_dxpl_checksum(hid_t dxpl_id, uint64_t cs) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ @@ -995,7 +995,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_dxpl_checksum(hid_t dxpl_id, uint32_t *cs/*out*/) +H5Pget_dxpl_checksum(hid_t dxpl_id, uint64_t *cs/*out*/) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1031,7 +1031,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pset_dxpl_checksum_ptr(hid_t dxpl_id, uint32_t *cs) +H5Pset_dxpl_checksum_ptr(hid_t dxpl_id, uint64_t *cs) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1069,7 +1069,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_dxpl_checksum_ptr(hid_t dxpl_id, uint32_t **cs/*out*/) +H5Pget_dxpl_checksum_ptr(hid_t dxpl_id, uint64_t **cs/*out*/) { H5P_genplist_t *plist = NULL; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2945,13 +2945,13 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id, char fake_char; int *status = NULL; H5VL_iod_io_info_t *info; /* info struct used to manage I/O parameters once the operation completes*/ - uint32_t internal_cs; /* internal checksum calculated in this function */ + uint64_t internal_cs; /* internal checksum calculated in this function */ size_t *vl_string_len = NULL; /* array that will contain lengths of strings if the datatype is a VL string type */ H5VL_iod_request_t **parent_reqs = NULL; size_t num_parents = 0; hid_t trans_id; H5TR_t *tr = NULL; - uint32_t user_cs; + uint64_t user_cs; uint32_t raw_cs_scope = 0; herr_t ret_value = SUCCEED; @@ -3062,7 +3062,7 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id, if((raw_cs_scope & H5_CHECKSUM_MEMORY) && user_cs && user_cs != internal_cs) { - fprintf(stderr, "Errrr.. In memory Data corruption. expecting %u, got %u\n", + fprintf(stderr, "Errrr.. In memory Data corruption. expecting %llu, got %llu\n", user_cs, internal_cs); HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "Checksum verification failed"); } @@ -4155,6 +4155,19 @@ H5VL_iod_attribute_read(void *_attr, hid_t type_id, void *buf, hid_t dxpl_id, vo attr->common.request = NULL; } + /* MSC - VLEN datatypes for attributes are not supported for now. */ + { + H5T_class_t class; + H5T_t *dt = NULL; + + if(NULL == (dt = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5T_NO_CLASS, "not a datatype") + + class = H5T_get_class(dt, FALSE); + if(H5T_VLEN == class || (H5T_STRING == class && H5T_is_variable_str(dt))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "VLEN datatypes for attributes not supported"); + } + /* calculate the size of the buffer needed */ size = H5Sget_simple_extent_npoints(attr->remote_attr.space_id) * H5Tget_size(type_id); @@ -4235,8 +4248,8 @@ H5VL_iod_attribute_write(void *_attr, hid_t type_id, const void *buf, hid_t dxpl hg_bulk_t *bulk_handle = NULL; int *status = NULL; size_t size; + uint64_t internal_cs; /* internal checksum calculated in this function */ H5VL_iod_io_info_t *info; - uint32_t cs; size_t num_parents = 0; hid_t trans_id; H5TR_t *tr = NULL; @@ -4262,14 +4275,24 @@ H5VL_iod_attribute_write(void *_attr, hid_t type_id, const void *buf, hid_t dxpl attr->common.request = NULL; } + /* MSC - VLEN datatypes for attributes are not supported for now. */ + { + H5T_class_t class; + H5T_t *dt = NULL; + + if(NULL == (dt = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5T_NO_CLASS, "not a datatype") + + class = H5T_get_class(dt, FALSE); + if(H5T_VLEN == class || (H5T_STRING == class && H5T_is_variable_str(dt))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "VLEN datatypes for attributes not supported"); + } + /* calculate the size of the buffer needed */ size = H5Sget_simple_extent_npoints(attr->remote_attr.space_id) * H5Tget_size(type_id); /* calculate a checksum for the data */ - cs = H5_checksum_fletcher32(buf, size); - - /* MSC- store it in a global variable for now so that the read can see it (for demo purposes */ - printf("Checksum Generated for attribute data at client: %u\n", cs); + internal_cs = H5_checksum_crc64(buf, size); /* allocate a bulk data transfer handle */ if(NULL == (bulk_handle = (hg_bulk_t *)H5MM_malloc(sizeof(hg_bulk_t)))) @@ -4298,6 +4321,7 @@ H5VL_iod_attribute_write(void *_attr, hid_t type_id, const void *buf, hid_t dxpl input.space_id = attr->remote_attr.space_id; input.trans_num = tr->trans_num; input.rcxt_num = tr->c_version; + input.checksum = internal_cs; input.cs_scope = attr->common.file->md_integrity_scope; status = (int *)malloc(sizeof(int)); @@ -6679,7 +6703,7 @@ H5VL_iod_map_set(void *_map, hid_t key_mem_type_id, const void *key, size_t num_parents = 0; H5TR_t *tr = NULL; hg_bulk_t *value_handle = NULL; - uint32_t key_cs, value_cs, user_cs; + uint64_t key_cs, value_cs, user_cs; uint32_t raw_cs_scope; H5VL_iod_request_t **parent_reqs = NULL; H5T_class_t val_type_class; @@ -6731,7 +6755,7 @@ H5VL_iod_map_set(void *_map, hid_t key_mem_type_id, const void *key, if((raw_cs_scope & H5_CHECKSUM_MEMORY) && user_cs && user_cs != value_cs) { - fprintf(stderr, "Errrr.. In memory Data corruption. expecting %u, got %u\n", + fprintf(stderr, "Errrr.. In memory Data corruption. expecting %llu, got %llu\n", user_cs, value_cs); HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "Checksum verification failed"); } @@ -6819,7 +6843,7 @@ H5VL_iod_map_get(void *_map, hid_t key_mem_type_id, const void *key, size_t key_size, val_size; hg_bulk_t *value_handle = NULL; hg_bulk_t dummy_handle; - uint32_t key_cs = 0; + uint64_t key_cs = 0; H5RC_t *rc = NULL; size_t num_parents = 0; hbool_t val_is_vl; @@ -7046,7 +7070,7 @@ H5VL_iod_map_exists(void *_map, hid_t key_mem_type_id, const void *key, size_t key_size; H5RC_t *rc = NULL; size_t num_parents = 0; - uint32_t key_cs = 0; + uint64_t key_cs = 0; H5VL_iod_request_t **parent_reqs = NULL; H5VL_iod_exists_info_t *info = NULL; herr_t ret_value = SUCCEED; @@ -7125,7 +7149,7 @@ H5VL_iod_map_delete(void *_map, hid_t key_mem_type_id, const void *key, int *status = NULL; size_t num_parents = 0; H5TR_t *tr = NULL; - uint32_t key_cs = 0; + uint64_t key_cs = 0; H5VL_iod_request_t **parent_reqs = NULL; herr_t ret_value = SUCCEED; diff --git a/src/H5VLiod.h b/src/H5VLiod.h index ec7039b..31eff65 100644 --- a/src/H5VLiod.h +++ b/src/H5VLiod.h @@ -30,14 +30,13 @@ #include "mercury.h" #include "mercury_handler.h" #include "mercury_macros.h" -#include "mercury_proc.h" +//#include "mercury_proc.h" #include "mercury_proc_string.h" -#include "mercury_error.h" -#include "mercury_bulk.h" -#include "mercury_config.h" +//#include "mercury_error.h" +//#include "mercury_bulk.h" +//#include "mercury_config.h" #include "na_mpi.h" -#include "na.h" - +//#include "na.h" #define H5VL_IOD (H5VL_iod_init()) #else @@ -62,10 +61,10 @@ H5_DLL herr_t EFF_start_server(MPI_Comm comm, MPI_Info info); H5_DLL herr_t EFF_init(MPI_Comm comm, MPI_Info info); H5_DLL herr_t EFF_finalize(void); -H5_DLL herr_t H5Pset_dxpl_checksum(hid_t dxpl_id, uint32_t value); -H5_DLL herr_t H5Pget_dxpl_checksum(hid_t dxpl_id, uint32_t *value); -H5_DLL herr_t H5Pset_dxpl_checksum_ptr(hid_t dxpl_id, uint32_t *value); -H5_DLL herr_t H5Pget_dxpl_checksum_ptr(hid_t dxpl_id, uint32_t **value); +H5_DLL herr_t H5Pset_dxpl_checksum(hid_t dxpl_id, uint64_t value); +H5_DLL herr_t H5Pget_dxpl_checksum(hid_t dxpl_id, uint64_t *value); +H5_DLL herr_t H5Pset_dxpl_checksum_ptr(hid_t dxpl_id, uint64_t *value); +H5_DLL herr_t H5Pget_dxpl_checksum_ptr(hid_t dxpl_id, uint64_t **value); H5_DLL herr_t H5Pset_metadata_integrity_scope(hid_t fapl_id, uint32_t scope); H5_DLL herr_t H5Pget_metadata_integrity_scope(hid_t fapl_id, uint32_t *scope); H5_DLL herr_t H5Pset_rawdata_integrity_scope(hid_t dxpl_id, uint32_t scope); diff --git a/src/H5VLiod_analysis.c b/src/H5VLiod_analysis.c index 5bd7a5d..d542ff3 100644 --- a/src/H5VLiod_analysis.c +++ b/src/H5VLiod_analysis.c @@ -701,7 +701,7 @@ H5VL__iod_read_selection(iod_handle_t coh, iod_obj_id_t obj_id, /* read the data selection from IOD. */ /* MSC - will need to do it in pieces, not it one shot. */ if(H5VL__iod_server_final_io(coh, obj_oh, space_id, type_id, - FALSE, buf, buf_size, 0, 0, rtid) < 0) + FALSE, buf, buf_size, (uint64_t)0, 0, rtid) < 0) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read from array object"); done: diff --git a/src/H5VLiod_attr.c b/src/H5VLiod_attr.c index c15d552..1853471 100644 --- a/src/H5VLiod_attr.c +++ b/src/H5VLiod_attr.c @@ -114,7 +114,7 @@ H5VL_iod_server_attr_create_cb(AXE_engine_t UNUSED axe_engine, /* set scratch pad in attribute */ if(cs_scope & H5_CHECKSUM_IOD) { - sp_cs = H5checksum(&sp, sizeof(sp), NULL); + sp_cs = H5_checksum_crc64(&sp, sizeof(sp)); if (iod_obj_set_scratch(attr_oh.wr_oh, wtid, &sp, &sp_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad"); } @@ -490,7 +490,7 @@ H5VL_iod_server_attr_read_cb(AXE_engine_t UNUSED axe_engine, /* MSC - NEED IOD */ #if 0 - attr_cs = H5checksum(buf, size, NULL); + attr_cs = H5_checksum_crc64(buf, size); if(attr_cs != iod_cs) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data corruption detected when reading attribute"); #endif @@ -672,7 +672,7 @@ H5VL_iod_server_attr_write_cb(AXE_engine_t UNUSED axe_engine, /* set the file descriptor */ file_desc = hslabs; - attr_cs = H5checksum(buf, size, NULL); + attr_cs = H5_checksum_crc64(buf, size); /* write from array object */ if(iod_array_write(iod_oh, wtid, NULL, mem_desc, &file_desc, &attr_cs, NULL) < 0) diff --git a/src/H5VLiod_client.c b/src/H5VLiod_client.c index dfa1d32..0f004a3 100644 --- a/src/H5VLiod_client.c +++ b/src/H5VLiod_client.c @@ -47,8 +47,7 @@ H5FL_EXTERN(H5VL_iod_dtype_t); typedef struct { size_t buf_size; uint8_t *buf_ptr; - uint32_t checksum; - uint8_t **off; + void **off; size_t *len; hsize_t curr_seq; size_t *str_len; /* used only for VL strings */ @@ -677,7 +676,7 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) req->state = H5VL_IOD_COMPLETED; } else { - uint32_t internal_cs = 0; + uint64_t internal_cs = 0; uint32_t raw_cs_scope = info->raw_cs_scope; /* calculate a checksum for the data recieved */ @@ -687,7 +686,8 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) /* verify data integrity */ if((raw_cs_scope & H5_CHECKSUM_TRANSFER) && internal_cs != read_status->cs) { - fprintf(stderr, "Errrrr! Dataset Read integrity failure (expecting %u got %u).\n", + fprintf(stderr, + "Errrrr! Dataset Read integrity failure (expecting %llu got %llu).\n", read_status->cs, internal_cs); req->status = H5ES_STATUS_FAIL; req->state = H5VL_IOD_COMPLETED; @@ -732,7 +732,7 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) dset_io_in_t input; void *read_buf; H5VL_iod_dset_t *dset = (H5VL_iod_dset_t *)req->obj; - uint32_t internal_cs = 0; + uint64_t internal_cs = 0; size_t buf_size = status->buf_size; hid_t rcxt_id; H5RC_t *rc; @@ -788,7 +788,7 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) } /* calculate a checksum for the data recieved */ - internal_cs = H5_checksum_lookup4(read_buf, buf_size, NULL); + internal_cs = H5_checksum_crc64(read_buf, buf_size); /* scatter the data into the user's buffer */ if(H5VL__iod_vl_read_finalize(buf_size, read_buf, (void *)info->buf_ptr, @@ -800,7 +800,8 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) /* verify data integrity */ if(internal_cs != vl_status.cs) { - fprintf(stderr, "Errrrr! Dataset Read integrity failure (expecting %u got %u).\n", + fprintf(stderr, + "Errrrr! Dataset Read integrity failure (expecting %llu got %llu).\n", internal_cs, status->cs); req->status = H5ES_STATUS_FAIL; req->state = H5VL_IOD_COMPLETED; @@ -913,16 +914,17 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) req->state = H5VL_IOD_COMPLETED; } else { - uint32_t internal_cs = 0; + uint64_t internal_cs = 0; uint32_t raw_cs_scope = info->raw_cs_scope; /* calculate a checksum for the data recieved */ - internal_cs = H5_checksum_lookup4(info->val_ptr, info->val_size, NULL); + internal_cs = H5_checksum_crc64(info->val_ptr, info->val_size); /* verify data integrity */ if((raw_cs_scope & H5_CHECKSUM_TRANSFER) && internal_cs != output->val_cs) { - fprintf(stderr, "Errrrr! MAP Get integrity failure (expecting %u got %u).\n", + fprintf(stderr, + "Errrrr! MAP Get integrity failure (expecting %llu got %llu).\n", output->val_cs, internal_cs); req->status = H5ES_STATUS_FAIL; req->state = H5VL_IOD_COMPLETED; @@ -2136,11 +2138,11 @@ H5VL__iod_pre_write_cb(void UNUSED *elem, hid_t type_id, unsigned UNUSED ndim, udata->str_len[i] = HDstrlen(buf[i]) + 1; udata->buf_size += udata->str_len[i] + sizeof(size_t); - udata->off[udata->curr_seq] = (uint8_t *)(udata->str_len+i); + udata->off[udata->curr_seq] = (void *)(udata->str_len+i); udata->len[udata->curr_seq] = sizeof(size_t); udata->curr_seq ++; - udata->off[udata->curr_seq] = (uint8_t*)buf[i]; + udata->off[udata->curr_seq] = (void *)buf[i]; udata->len[udata->curr_seq] = udata->str_len[i]; udata->curr_seq ++; @@ -2160,11 +2162,11 @@ H5VL__iod_pre_write_cb(void UNUSED *elem, hid_t type_id, unsigned UNUSED ndim, elmt_size = H5T_get_size(super) * vl->len; udata->buf_size += elmt_size + sizeof(size_t); - udata->off[udata->curr_seq] = (uint8_t *)udata->buf_ptr; + udata->off[udata->curr_seq] = (void *)udata->buf_ptr; udata->len[udata->curr_seq] = sizeof(size_t); udata->curr_seq ++; - udata->off[udata->curr_seq] = (uint8_t *)(vl->p); + udata->off[udata->curr_seq] = (void *)(vl->p); udata->len[udata->curr_seq] = elmt_size; udata->curr_seq ++; @@ -2201,12 +2203,12 @@ done: */ herr_t H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, - /*out*/uint32_t *_checksum, + /*out*/uint64_t *_checksum, /*out*/hg_bulk_t *bulk_handle, /*out*/size_t **vl_str_len) { hsize_t buf_size = 0; - uint32_t checksum = 0; + uint64_t checksum = 0; H5S_t *space = NULL; H5T_t *dt = NULL; size_t nelmts; @@ -2231,7 +2233,6 @@ H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, if(H5T_is_variable_str(dt)) { char bogus; /* bogus value to pass to H5Diterate() */ H5VL_iod_pre_write_t udata; - H5_checksum_seed_t cs; hg_bulk_segment_t *bulk_segments = NULL; unsigned u; @@ -2241,8 +2242,7 @@ H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, /* set H5Diterate op_data */ udata.buf_size = 0; udata.buf_ptr = (uint8_t *)buf; - udata.checksum = 0; - udata.off = (uint8_t **)malloc(nelmts * 2 * sizeof(uint8_t *)); + udata.off = (void **)malloc(nelmts * 2 * sizeof(void *)); udata.len = (size_t *)malloc(nelmts * 2 * sizeof(size_t)); udata.curr_seq = 0; @@ -2258,7 +2258,7 @@ H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, sizeof(hg_bulk_segment_t)); for (u = 0; u <udata.curr_seq ; u++) { - bulk_segments[u].address = (void *)udata.off[u]; + bulk_segments[u].address = udata.off[u]; bulk_segments[u].size = udata.len[u]; } @@ -2270,14 +2270,9 @@ H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, free(bulk_segments); bulk_segments = NULL; - if(_checksum) { - /* compute checksum of length array and the actual stings array */ - cs.a = cs.b = cs.c = cs.state = 0; - cs.total_length = buf_size; - for(u = 0; u < udata.curr_seq ; u++) { - checksum = H5_checksum_lookup4(udata.off[u], udata.len[u], &cs); - } - } + if(_checksum) + checksum = H5_checksum_crc64_fragments(udata.off, udata.len, + (size_t)udata.curr_seq); /* cleanup */ if(udata.curr_seq) { @@ -2369,14 +2364,12 @@ H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, { char bogus; /* bogus value to pass to H5Diterate() */ H5VL_iod_pre_write_t udata; - H5_checksum_seed_t cs; hg_bulk_segment_t *bulk_segments = NULL; unsigned u; udata.buf_size = 0; udata.buf_ptr = (uint8_t *)buf; - udata.checksum = 0; - udata.off = (uint8_t **)malloc(nelmts * 2 * sizeof(uint8_t *)); + udata.off = (void **)malloc(nelmts * 2 * sizeof(void *)); udata.len = (size_t *)malloc(nelmts * 2 * sizeof(size_t)); udata.curr_seq = 0; @@ -2391,7 +2384,7 @@ H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, bulk_segments = (hg_bulk_segment_t *)malloc((size_t)udata.curr_seq * sizeof(hg_bulk_segment_t)); for (u = 0; u < udata.curr_seq ; u++) { - bulk_segments[u].address = (void *)udata.off[u]; + bulk_segments[u].address = udata.off[u]; bulk_segments[u].size = udata.len[u]; } @@ -2404,13 +2397,9 @@ H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, free(bulk_segments); bulk_segments = NULL; - cs.a = cs.b = cs.c = cs.state = 0; - cs.total_length = buf_size; - if(_checksum) { - for(u = 0; u < udata.curr_seq ; u++) { - checksum = H5_checksum_lookup4(udata.off[u], udata.len[u], &cs); - } + checksum = H5_checksum_crc64_fragments(udata.off, udata.len, + (size_t)udata.curr_seq); } /* cleanup */ @@ -2746,7 +2735,7 @@ done: */ herr_t H5VL_iod_map_get_size(hid_t type_id, const void *buf, - /*out*/uint32_t *checksum, + /*out*/uint64_t *checksum, /*out*/size_t *size, /*out*/H5T_class_t *dt_class) { size_t buf_size = 0; @@ -2768,7 +2757,7 @@ H5VL_iod_map_get_size(hid_t type_id, const void *buf, buf_size = HDstrlen((const char*)buf) + 1; /* compute checksum */ - *checksum = H5_checksum_lookup4(buf, buf_size, NULL); + *checksum = H5_checksum_crc64(buf, buf_size); break; } case H5T_INTEGER: @@ -2789,7 +2778,7 @@ H5VL_iod_map_get_size(hid_t type_id, const void *buf, buf_size = H5T_get_size(dt); /* compute checksum */ - *checksum = H5_checksum_lookup4(buf, buf_size, NULL); + *checksum = H5_checksum_crc64(buf, buf_size); break; /* If this is a variable length datatype, iterate over it */ @@ -2806,7 +2795,7 @@ H5VL_iod_map_get_size(hid_t type_id, const void *buf, buf_size = H5T_get_size(super) * vl->len; /* compute checksum */ - *checksum = H5_checksum_lookup4(vl->p, buf_size, NULL); + *checksum = H5_checksum_crc64(vl->p, buf_size); H5T_close(super); break; } diff --git a/src/H5VLiod_client.h b/src/H5VLiod_client.h index c066a08..80211a5 100644 --- a/src/H5VLiod_client.h +++ b/src/H5VLiod_client.h @@ -282,7 +282,7 @@ typedef struct H5VL_iod_io_info_t { size_t nelmts; size_t type_size; struct H5S_t *space; - uint32_t *cs_ptr; + uint64_t *cs_ptr; uint32_t raw_cs_scope; hid_t file_space_id; hid_t mem_type_id; @@ -307,7 +307,7 @@ typedef struct H5VL_iod_map_io_info_t { hg_bulk_t *value_handle; size_t val_size; uint32_t raw_cs_scope; - uint32_t *val_cs_ptr; + uint64_t *val_cs_ptr; hid_t val_mem_type_id; hid_t key_mem_type_id; hid_t dxpl_id; @@ -355,12 +355,12 @@ H5_DLL herr_t H5VL__iod_create_and_forward(hg_id_t op_id, H5RQ_type_t op_type, H5_DLL herr_t H5VL_iod_map_dtype_info(hid_t type_id, /*out*/ hbool_t *is_vl, /*out*/size_t *size); H5_DLL herr_t H5VL_iod_map_get_size(hid_t type_id, const void *buf, - /*out*/uint32_t *checksum, + /*out*/uint64_t *checksum, /*out*/size_t *size, /*out*/H5T_class_t *dt_class); H5_DLL herr_t H5VL_iod_gen_obj_id(int myrank, int nranks, uint64_t cur_index, iod_obj_type_t type, uint64_t *id); H5_DLL herr_t H5VL_iod_pre_write(hid_t type_id, hid_t space_id, const void *buf, - /*out*/uint32_t *_checksum, + /*out*/uint64_t *_checksum, /*out*/hg_bulk_t *bulk_handle, /*out*/size_t **vl_str_len); H5_DLL herr_t H5VL_iod_pre_read(hid_t type_id, hid_t space_id, const void *buf, diff --git a/src/H5VLiod_common.h b/src/H5VLiod_common.h index 34d0162..a9c3b5e 100644 --- a/src/H5VLiod_common.h +++ b/src/H5VLiod_common.h @@ -40,7 +40,7 @@ typedef enum H5VL_iod_state_t { typedef struct H5VL_iod_read_status_t { int ret; - uint32_t cs; + uint64_t cs; size_t buf_size; } H5VL_iod_read_status_t; @@ -167,7 +167,7 @@ MERCURY_GEN_PROC(attr_io_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope)) ((uint64_t)(rcxt_num)) ((uint64_t)(trans_num)) ((iod_handle_t)(coh)) ((iod_handles_t)(iod_oh)) ((iod_obj_id_t)(iod_id)) ((iod_obj_id_t)(mdkv_id)) - ((hid_t)(space_id)) + ((uint64_t)(checksum)) ((hid_t)(space_id)) ((hid_t)(type_id)) ((hg_bulk_t)(bulk_handle))) MERCURY_GEN_PROC(attr_close_in_t, ((axe_t)(axe_info)) ((iod_handles_t)(iod_oh)) ((iod_obj_id_t)(iod_id))) @@ -215,7 +215,7 @@ MERCURY_GEN_PROC(map_set_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope)) ((iod_obj_id_t)(iod_id)) ((hid_t)(key_maptype_id)) ((hid_t)(key_memtype_id)) ((binary_buf_t)(key)) ((hid_t)(val_maptype_id)) ((hid_t)(val_memtype_id)) - ((uint32_t)(val_checksum)) ((hg_bulk_t)(val_handle)) + ((uint64_t)(val_checksum)) ((hg_bulk_t)(val_handle)) ((hid_t)(dxpl_id))) MERCURY_GEN_PROC(map_get_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope)) ((uint64_t)(rcxt_num)) @@ -225,7 +225,7 @@ MERCURY_GEN_PROC(map_get_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope)) ((hid_t)(val_maptype_id)) ((hid_t)(val_memtype_id)) ((hbool_t)(val_is_vl)) ((hg_bulk_t)(val_handle)) ((size_t)(val_size)) ((hid_t)(dxpl_id))) -MERCURY_GEN_PROC(map_get_out_t, ((int32_t)(ret)) ((size_t)(val_size)) ((uint32_t)(val_cs))) +MERCURY_GEN_PROC(map_get_out_t, ((int32_t)(ret)) ((size_t)(val_size)) ((uint64_t)(val_cs))) MERCURY_GEN_PROC(map_get_count_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope)) ((uint64_t)(rcxt_num)) ((iod_handle_t)(coh)) ((iod_handles_t)(iod_oh)) @@ -265,7 +265,7 @@ MERCURY_GEN_PROC(dset_io_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope)) ((iod_handle_t)(coh)) ((iod_handles_t)(iod_oh)) ((iod_obj_id_t)(iod_id)) ((iod_obj_id_t)(mdkv_id)) ((hid_t)(dset_type_id)) ((hid_t)(mem_type_id)) - ((hid_t)(space_id)) ((hid_t)(dxpl_id)) ((uint32_t)(checksum)) + ((hid_t)(space_id)) ((hid_t)(dxpl_id)) ((uint64_t)(checksum)) ((hg_bulk_t)(bulk_handle))) MERCURY_GEN_PROC(dset_get_vl_size_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope)) ((uint64_t)(rcxt_num)) @@ -273,7 +273,7 @@ MERCURY_GEN_PROC(dset_get_vl_size_in_t, ((axe_t)(axe_info)) ((uint32_t)(cs_scope ((iod_obj_id_t)(iod_id)) ((iod_obj_id_t)(mdkv_id)) ((hid_t)(mem_type_id)) ((hid_t)(space_id)) ((hid_t)(dxpl_id))) -MERCURY_GEN_PROC(dset_read_out_t, ((int32_t)(ret)) ((uint32_t)(cs)) ((size_t)(buf_size))) +MERCURY_GEN_PROC(dset_read_out_t, ((int32_t)(ret)) ((uint64_t)(cs)) ((size_t)(buf_size))) MERCURY_GEN_PROC(dset_close_in_t, ((axe_t)(axe_info)) ((iod_handles_t)(iod_oh)) ((iod_obj_id_t)(iod_id))) diff --git a/src/H5VLiod_dset.c b/src/H5VLiod_dset.c index f68aedc..73a0ecc 100644 --- a/src/H5VLiod_dset.c +++ b/src/H5VLiod_dset.c @@ -169,7 +169,7 @@ H5VL_iod_server_dset_create_cb(AXE_engine_t UNUSED axe_engine, if(cs_scope & H5_CHECKSUM_IOD) { iod_checksum_t sp_cs; - sp_cs = H5checksum(&sp, sizeof(sp), NULL); + sp_cs = H5_checksum_crc64(&sp, sizeof(sp)); if (iod_obj_set_scratch(dset_oh.wr_oh, wtid, &sp, &sp_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad"); } @@ -450,7 +450,7 @@ H5VL_iod_server_dset_read_cb(AXE_engine_t UNUSED axe_engine, hg_bulk_request_t bulk_request; /* HG request */ size_t size, buf_size; void *buf = NULL; /* buffer to hold outgoing data */ - uint32_t cs = 0; /* checksum value */ + iod_checksum_t cs = 0; /* checksum value */ uint32_t raw_cs_scope; hbool_t is_vl_data; size_t nelmts; /* number of elements selected to read */ @@ -499,7 +499,7 @@ H5VL_iod_server_dset_read_cb(AXE_engine_t UNUSED axe_engine, if(!is_vl_data) { /* If the data is not VL, we can read the data from the array the normal way */ if(H5VL__iod_server_final_io(coh, iod_oh.rd_oh, space_id, src_id, - FALSE, buf, buf_size, 0, raw_cs_scope, rtid) < 0) { + FALSE, buf, buf_size, (uint64_t)0, raw_cs_scope, rtid) < 0) { fprintf(stderr, "can't read from array object\n"); ret_value = FAIL; goto done; @@ -520,7 +520,7 @@ H5VL_iod_server_dset_read_cb(AXE_engine_t UNUSED axe_engine, if(raw_cs_scope) { /* calculate a checksum for the data to be sent */ - cs = H5checksum(buf, size, NULL); + cs = H5_checksum_crc64(buf, size); } #if H5VL_IOD_DEBUG else { @@ -592,7 +592,7 @@ H5VL_iod_server_dset_read_cb(AXE_engine_t UNUSED axe_engine, } if(!(raw_cs_scope & H5_CHECKSUM_NONE)) { /* calculate a checksum for the data to be sent */ - cs = H5checksum(buf, buf_size, NULL); + cs = H5_checksum_crc64(buf, buf_size); } } @@ -619,7 +619,7 @@ done: HDONE_ERROR(H5E_SYM, H5E_WRITEERROR, FAIL, "can't send result of write to client"); #if H5VL_IOD_DEBUG - fprintf(stderr, "Done with dset read, checksum %u, sending response to client\n", cs); + fprintf(stderr, "Done with dset read, checksum %llu, sending response to client\n", cs); #endif input = (dset_io_in_t *)H5MM_xfree(input); @@ -788,7 +788,7 @@ H5VL_iod_server_dset_get_vl_size_cb(AXE_engine_t UNUSED axe_engine, #if 0 /* Verify checksum for that entry */ buf_ptr = (uint8_t *)buf; - entry_cs = H5checksum(buf_ptr, sizeof(iod_size_t) + sizeof(iod_obj_id_t), NULL); + entry_cs = H5_checksum_crc64(buf_ptr, sizeof(iod_size_t) + sizeof(iod_obj_id_t)); if(entry_cs != io_array[n].cs) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data Corruption detected when reading"); buf_ptr += sizeof(iod_size_t) + sizeof(iod_obj_id_t); @@ -883,7 +883,7 @@ H5VL_iod_server_dset_write_cb(AXE_engine_t UNUSED axe_engine, iod_obj_id_t iod_id = input->iod_id; /* dset ID */ hg_bulk_t bulk_handle = input->bulk_handle; /* bulk handle for data */ hid_t space_id = input->space_id; /* file space selection */ - uint32_t cs = input->checksum; /* checksum recieved for data */ + uint64_t cs = input->checksum; /* checksum recieved for data */ hid_t src_id = input->mem_type_id; /* the memory type of the elements */ hid_t dst_id = input->dset_type_id; /* the datatype of the dataset's element */ iod_trans_id_t wtid = input->trans_num; @@ -894,7 +894,7 @@ H5VL_iod_server_dset_write_cb(AXE_engine_t UNUSED axe_engine, hg_bulk_request_t bulk_request; /* HG request */ size_t size, buf_size; hbool_t is_vl_data; - uint32_t data_cs = 0; + iod_checksum_t data_cs = 0; uint32_t raw_cs_scope; unsigned u; void *buf = NULL; @@ -955,9 +955,9 @@ H5VL_iod_server_dset_write_cb(AXE_engine_t UNUSED axe_engine, /* verify data if transfer flag is set */ if(raw_cs_scope & H5_CHECKSUM_TRANSFER) { - data_cs = H5checksum(buf, size, NULL); + data_cs = H5_checksum_crc64(buf, size); if(cs != data_cs) { - fprintf(stderr, "Errrr.. Network transfer Data corruption. expecting %u, got %u\n", + fprintf(stderr, "Errrr.. Network transfer Data corruption. expecting %llu, got %llu\n", cs, data_cs); ret_value = FAIL; goto done; @@ -1213,7 +1213,7 @@ done: herr_t H5VL__iod_server_final_io(iod_handle_t coh, iod_handle_t iod_oh, hid_t space_id, hid_t type_id, hbool_t write_op, void *buf, - size_t buf_size, uint32_t cs, uint32_t cs_scope, + size_t buf_size, iod_checksum_t cs, uint32_t cs_scope, iod_trans_id_t tid) { int ndims, i; /* dataset's rank/number of dimensions */ @@ -1299,7 +1299,7 @@ H5VL__iod_server_final_io(iod_handle_t coh, iod_handle_t iod_oh, hid_t space_id, /* If this is a write op, compute the checksum for each memory fragment */ if(write_op && (cs_scope & H5_CHECKSUM_IOD)) - cs_list[n] = H5checksum(buf_ptr, (size_t)num_bytes, NULL); + cs_list[n] = H5_checksum_crc64(buf_ptr, (size_t)num_bytes); buf_ptr += num_bytes; @@ -1354,7 +1354,7 @@ H5VL__iod_server_final_io(iod_handle_t coh, iod_handle_t iod_oh, hid_t space_id, if(!write_op && (cs_scope & H5_CHECKSUM_IOD)) { hsize_t num_bytes = 0; hsize_t num_elems = 1; - uint32_t checksum; + iod_checksum_t checksum; buf_ptr = (uint8_t *)buf; @@ -1364,7 +1364,7 @@ H5VL__iod_server_final_io(iod_handle_t coh, iod_handle_t iod_oh, hid_t space_id, num_elems *= (hslabs[n].count[i] * hslabs[n].block[i]); num_bytes = num_elems * elmt_size; - checksum = H5checksum(buf_ptr, (size_t)num_bytes, NULL); + checksum = H5_checksum_crc64(buf_ptr, (size_t)num_bytes); if(checksum != cs_list[n]) { fprintf(stderr, "Data Corruption detected when reading\n"); @@ -1502,7 +1502,6 @@ H5VL__iod_server_vl_data_io_cb(void UNUSED *elem, hid_t type_id, unsigned ndims, size_t old_seq_len = 0; unsigned u; iod_checksum_t entry_cs = 0, read_cs = 0; - H5_checksum_seed_t cs; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -1536,11 +1535,17 @@ H5VL__iod_server_vl_data_io_cb(void UNUSED *elem, hid_t type_id, unsigned ndims, /* MSC - NEED IOD */ #if 0 - /* compute checksum of blob ID and sequence length */ - cs.a = cs.b = cs.c = cs.state = 0; - cs.total_length = sizeof(iod_obj_id_t) + sizeof(iod_size_t); - entry_cs = H5checksum(&blob_id, sizeof(iod_obj_id_t), &cs); - entry_cs = H5checksum(&seq_len, sizeof(iod_size_t), &cs); + { + void *buffers[2]; + size_t buf_sizes[2]; + + buffers[0] = &blob_id; + buf_sizes[0] = sizeof(iod_obj_id_t); + buffers[1] = &seq_len; + buf_sizes[1] = sizeof(iod_size_t); + + entry_cs = H5_checksum_crc64_fragments(buffers, buf_sizes, 2); + } if(entry_cs != read_cs) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data Corruption detected when reading"); @@ -1622,10 +1627,17 @@ H5VL__iod_server_vl_data_io_cb(void UNUSED *elem, hid_t type_id, unsigned ndims, mem_desc->frag[1].len = sizeof(iod_size_t); /* compute checksum of blob ID and sequence length */ - cs.a = cs.b = cs.c = cs.state = 0; - cs.total_length = sizeof(iod_obj_id_t) + sizeof(iod_size_t); - entry_cs = H5checksum(&blob_id, sizeof(iod_obj_id_t), &cs); - entry_cs = H5checksum(&seq_len, sizeof(iod_size_t), &cs); + { + void *buffers[2]; + size_t buf_sizes[2]; + + buffers[0] = &blob_id; + buf_sizes[0] = sizeof(iod_obj_id_t); + buffers[1] = &seq_len; + buf_sizes[1] = sizeof(iod_size_t); + + entry_cs = H5_checksum_crc64_fragments(buffers, buf_sizes, 2); + } /* write the blob ID & size to the array element */ if(iod_array_write(udata->iod_oh, tid, NULL, diff --git a/src/H5VLiod_dtype.c b/src/H5VLiod_dtype.c index d291c44..1f1bc49 100644 --- a/src/H5VLiod_dtype.c +++ b/src/H5VLiod_dtype.c @@ -113,7 +113,7 @@ H5VL_iod_server_dtype_commit_cb(AXE_engine_t UNUSED axe_engine, if(cs_scope & H5_CHECKSUM_IOD) { iod_checksum_t sp_cs; - sp_cs = H5checksum(&sp, sizeof(sp), NULL); + sp_cs = H5_checksum_crc64(&sp, sizeof(sp)); if (iod_obj_set_scratch(dtype_oh.wr_oh, wtid, &sp, &sp_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad"); } @@ -153,7 +153,7 @@ H5VL_iod_server_dtype_commit_cb(AXE_engine_t UNUSED axe_engine, iod_checksum_t dt_cs; /* calculate a checksum for the datatype */ - dt_cs = H5checksum(buf, buf_size, NULL); + dt_cs = H5_checksum_crc64(buf, buf_size); /* write the serialized type value to the BLOB object */ if(iod_blob_write(dtype_oh.wr_oh, wtid, NULL, mem_desc, file_desc, &dt_cs, NULL) < 0) @@ -362,7 +362,7 @@ H5VL_iod_server_dtype_open_cb(AXE_engine_t UNUSED axe_engine, /* MSC - NEED IOD */ if(iod_cs && (cs_scope & H5_CHECKSUM_IOD)) { /* calculate a checksum for the datatype */ - dt_cs = H5checksum(buf, buf_size, NULL); + dt_cs = H5_checksum_crc64(buf, buf_size); /* Verifty checksum against one given by IOD */ if(iod_cs != dt_cs) diff --git a/src/H5VLiod_file.c b/src/H5VLiod_file.c index 44832db..3662182 100644 --- a/src/H5VLiod_file.c +++ b/src/H5VLiod_file.c @@ -124,7 +124,7 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine, if(cs_scope & H5_CHECKSUM_IOD) { iod_checksum_t sp_cs; - sp_cs = H5checksum(&sp, sizeof(sp), NULL); + sp_cs = H5_checksum_crc64(&sp, sizeof(sp)); /* set scratch pad in root group */ if (iod_obj_set_scratch(root_oh.wr_oh, first_tid, &sp, &sp_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad"); diff --git a/src/H5VLiod_group.c b/src/H5VLiod_group.c index 79ceef2..cd36529 100644 --- a/src/H5VLiod_group.c +++ b/src/H5VLiod_group.c @@ -109,7 +109,7 @@ H5VL_iod_server_group_create_cb(AXE_engine_t UNUSED axe_engine, if(cs_scope & H5_CHECKSUM_IOD) { iod_checksum_t sp_cs; - sp_cs = H5checksum(&sp, sizeof(sp), NULL); + sp_cs = H5_checksum_crc64(&sp, sizeof(sp)); if (iod_obj_set_scratch(grp_oh.wr_oh, wtid, &sp, &sp_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad"); } diff --git a/src/H5VLiod_map.c b/src/H5VLiod_map.c index f977d27..b6d7224 100644 --- a/src/H5VLiod_map.c +++ b/src/H5VLiod_map.c @@ -114,7 +114,7 @@ H5VL_iod_server_map_create_cb(AXE_engine_t UNUSED axe_engine, if(cs_scope & H5_CHECKSUM_IOD) { iod_checksum_t sp_cs; - sp_cs = H5checksum(&sp, sizeof(sp), NULL); + sp_cs = H5_checksum_crc64(&sp, sizeof(sp)); if (iod_obj_set_scratch(map_oh.wr_oh, wtid, &sp, &sp_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't set scratch pad"); } @@ -346,7 +346,7 @@ H5VL_iod_server_map_set_cb(AXE_engine_t UNUSED axe_engine, hid_t val_maptype_id = input->val_maptype_id; binary_buf_t key = input->key; hg_bulk_t value_handle = input->val_handle; /* bulk handle for data */ - uint32_t value_cs = input->val_checksum; /* checksum recieved for data */ + iod_checksum_t value_cs = input->val_checksum; /* checksum recieved for data */ hid_t dxpl_id = input->dxpl_id; iod_trans_id_t wtid = input->trans_num; iod_trans_id_t rtid = input->rcxt_num; @@ -406,11 +406,11 @@ H5VL_iod_server_map_set_cb(AXE_engine_t UNUSED axe_engine, /* verify data if transfer flag is set */ if(raw_cs_scope & H5_CHECKSUM_TRANSFER) { - uint32_t data_cs; + iod_checksum_t data_cs; - data_cs = H5checksum(val_buf, val_size, NULL); + data_cs = H5_checksum_crc64(val_buf, val_size); if(value_cs != data_cs) { - fprintf(stderr, "Errrr.. Network transfer Data corruption. expecting %u, got %u\n", + fprintf(stderr, "Errrr.. Network transfer Data corruption. expecting %llu, got %llu\n", value_cs, data_cs); ret_value = FAIL; goto done; @@ -648,7 +648,7 @@ H5VL_iod_server_map_get_cb(AXE_engine_t UNUSED axe_engine, output.val_size = src_size; if(raw_cs_scope) { /* calculate a checksum for the data to be sent */ - output.val_cs = H5checksum(val_buf, (size_t)src_size, NULL); + output.val_cs = H5_checksum_crc64(val_buf, (size_t)src_size); } #if H5VL_IOD_DEBUG else { @@ -696,7 +696,7 @@ H5VL_iod_server_map_get_cb(AXE_engine_t UNUSED axe_engine, if(raw_cs_scope) { /* calculate a checksum for the data to be sent */ - output.val_cs = H5checksum(val_buf, val_size, NULL); + output.val_cs = H5_checksum_crc64(val_buf, val_size); } #if H5VL_IOD_DEBUG else { diff --git a/src/H5VLiod_obj.c b/src/H5VLiod_obj.c index 19a9b54..a924e36 100644 --- a/src/H5VLiod_obj.c +++ b/src/H5VLiod_obj.c @@ -221,7 +221,7 @@ H5VL_iod_server_object_open_cb(AXE_engine_t UNUSED axe_engine, /* MSC - NEED IOD */ #if 0 /* calculate a checksum for the datatype */ - dt_cs = H5checksum(buf, buf_size, NULL); + dt_cs = H5_checksum_crc64(buf, buf_size); /* Verifty checksum against one given by IOD */ if(iod_cs != dt_cs) @@ -422,7 +422,7 @@ H5VL_iod_server_object_copy_cb(AXE_engine_t UNUSED axe_engine, /* MSC - NEED IOD */ /* calculate a checksum for the datatype */ - dt_cs = H5checksum(buf, buf_size, NULL); + dt_cs = H5_checksum_crc64(buf, buf_size); /* Verifty checksum against one given by IOD */ if(iod_cs != dt_cs) diff --git a/src/H5VLiod_server.h b/src/H5VLiod_server.h index 8a89c88..49d1eaf 100644 --- a/src/H5VLiod_server.h +++ b/src/H5VLiod_server.h @@ -410,7 +410,7 @@ H5_DLL herr_t H5VL__iod_server_adjust_buffer(hid_t from_type_id, hid_t to_type_i H5_DLL herr_t H5VL_iod_verify_scratch_pad(scratch_pad sp, iod_checksum_t iod_cs); H5_DLL herr_t H5VL__iod_server_final_io(iod_handle_t coh, iod_handle_t iod_oh, hid_t space_id, hid_t type_id, hbool_t write_op, void *buf, - size_t buf_size, uint32_t cs, uint32_t cs_scope, + size_t buf_size, iod_checksum_t cs, uint32_t cs_scope, iod_trans_id_t tid); #endif /* H5_HAVE_EFF */ diff --git a/src/H5VLiod_util.c b/src/H5VLiod_util.c index 52afbd5..5eef5fe 100644 --- a/src/H5VLiod_util.c +++ b/src/H5VLiod_util.c @@ -717,7 +717,7 @@ H5VL_iod_insert_new_link(iod_handle_t oh, iod_trans_id_t tid, const char *link_n { iod_kv_t kv; void *value = NULL; - const uint8_t *val_ptr = NULL; + uint8_t *val_ptr = NULL; size_t value_len; herr_t ret_value = SUCCEED; @@ -738,13 +738,13 @@ H5VL_iod_insert_new_link(iod_handle_t oh, iod_trans_id_t tid, const char *link_n } case H5L_TYPE_SOFT: { - value_len = sizeof(H5L_type_t) + strlen((char *)link_val) + 1; + value_len = sizeof(H5L_type_t) + strlen((const char *)link_val) + 1; value = malloc(value_len); val_ptr = (uint8_t *)value; memcpy(val_ptr, &link_type, sizeof(H5L_type_t)); - strcpy((char *)(val_ptr+sizeof(H5L_type_t)), (char *)link_val); + strcpy((char *)(val_ptr+sizeof(H5L_type_t)), (const char *)link_val); break; } @@ -1015,7 +1015,7 @@ H5VL_iod_verify_scratch_pad(scratch_pad sp, iod_checksum_t iod_cs) /* MSC - Need IOD */ #if 0 - computed_cs = H5checksum(&sp, sizeof(sp), NULL); + computed_cs = H5_checksum_crc64(&sp, sizeof(sp)); if(computed_cs != iod_cs) { fprintf(stderr, "Scratch pad integrity check failed. IOD cs = %u, Computed cs = %u", diff --git a/src/H5checksum.c b/src/H5checksum.c index a6a68c8..2b57d04 100644 --- a/src/H5checksum.c +++ b/src/H5checksum.c @@ -33,7 +33,7 @@ /* Headers */ /***********/ #include "H5private.h" /* Generic Functions */ - +#include "mchecksum.h" /* Mercury Checksum library */ /****************/ /* Local Macros */ @@ -664,3 +664,61 @@ H5_hash_string(const char *str) FUNC_LEAVE_NOAPI(hash) } /* end H5_hash_string() */ +uint64_t +H5_checksum_crc64(const void *buf, size_t buf_size) +{ + const char *hash_method = "crc64"; + size_t hash_size; + uint64_t hash; + mchecksum_object_t checksum; + + /* Initialize checksum */ + mchecksum_init(hash_method, &checksum); + + /* Update checksum */ + mchecksum_update(checksum, buf, buf_size); + + /* Get size of checksum */ + hash_size = mchecksum_get_size(checksum); + + assert(hash_size == sizeof(uint64_t)); + + /* get checksum value */ + mchecksum_get(checksum, &hash, hash_size, 1); + + /* Destroy checksum */ + mchecksum_destroy(checksum); + + return hash; +} + +uint64_t +H5_checksum_crc64_fragments(void **buf, size_t *buf_size, size_t count) +{ + const char *hash_method = "crc64"; + size_t hash_size; + uint64_t hash; + size_t i; + mchecksum_object_t checksum; + + /* Initialize checksum */ + mchecksum_init(hash_method, &checksum); + + /* Update checksum */ + for (i = 0; i < count; i++) { + mchecksum_update(checksum, buf[i], buf_size[i]); + } + + /* Get size of checksum */ + hash_size = mchecksum_get_size(checksum); + + assert(hash_size == sizeof(uint64_t)); + + /* get checksum value */ + mchecksum_get(checksum, &hash, hash_size, 1); + + /* Destroy checksum */ + mchecksum_destroy(checksum); + + return hash; +} diff --git a/src/H5public.h b/src/H5public.h index c4b50cf..e281f91 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -323,6 +323,9 @@ H5_DLL herr_t H5check_version(unsigned majnum, unsigned minnum, unsigned relnum); H5_DLL uint32_t H5checksum(const void *key, size_t length, H5_checksum_seed_t *cs); +H5_DLL uint64_t H5_checksum_crc64(const void *buf, size_t buf_size); +H5_DLL uint64_t H5_checksum_crc64_fragments(void **buf, size_t *buf_size, size_t count); + #ifdef __cplusplus } #endif |