From 53c4cce5f99ffa0dde6dd6a412bcb0a7a349f8c0 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 18 Sep 2013 08:53:19 -0500 Subject: [svn-r24152] add more data integrity checks. --- src/H5VLiod_attr.c | 16 ++++++++-------- src/H5VLiod_dset.c | 41 ++++++++++++++++++++++++++++++++++++----- src/H5VLiod_dtype.c | 23 +++++++++++++++++++---- src/H5VLiod_file.c | 4 ++-- src/H5VLiod_group.c | 4 ++-- src/H5VLiod_link.c | 6 +++--- src/H5VLiod_map.c | 4 ++-- src/H5VLiod_obj.c | 10 +++++----- 8 files changed, 77 insertions(+), 31 deletions(-) diff --git a/src/H5VLiod_attr.c b/src/H5VLiod_attr.c index eab7abe..5c96b7e 100644 --- a/src/H5VLiod_attr.c +++ b/src/H5VLiod_attr.c @@ -60,7 +60,7 @@ H5VL_iod_server_attr_create_cb(AXE_engine_t UNUSED axe_engine, iod_array_struct_t array; /* IOD array structure for attribute's creation */ iod_size_t *max_dims; /* MAX dims for IOD */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; iod_ret_t ret; hbool_t collective = FALSE; /* MSC - change when we allow for collective */ herr_t ret_value = SUCCEED; @@ -103,7 +103,7 @@ H5VL_iod_server_attr_create_cb(AXE_engine_t UNUSED axe_engine, /* for the process that succeeded in creating the attribute, update the parent scratch pad, create attribute scratch pad */ if(0 == ret) { - uint32_t sp_cs; + iod_checksum_t sp_cs; /* create the metadata KV object for the attribute */ if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, @@ -239,7 +239,7 @@ H5VL_iod_server_attr_open_cb(AXE_engine_t UNUSED axe_engine, const char *loc_name = input->path; /* current path to start traversal */ const char *attr_name = input->attr_name; /* attribute's name to open */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; H5VL_iod_link_t iod_link; herr_t ret_value = SUCCEED; @@ -410,7 +410,7 @@ H5VL_iod_server_attr_read_cb(AXE_engine_t UNUSED axe_engine, void *buf; /* buffer to hold outgoing data */ iod_handle_t mdkv_oh; /* metadata KV handle of attribute */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; int ndims; /* dataset's rank/number of dimensions */ hssize_t num_descriptors = 0; /* number of IOD file descriptors needed to describe filespace selection */ na_addr_t dest = HG_Handler_get_addr(op_data->hg_handle); /* destination address to push data to */ @@ -578,7 +578,7 @@ H5VL_iod_server_attr_write_cb(AXE_engine_t UNUSED axe_engine, void *buf; /* buffer to hold outgoing data */ int ndims; /* dataset's rank/number of dimensions */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; iod_handle_t mdkv_oh; /* metadata KV handle of attribute */ hssize_t num_descriptors = 0; /* number of IOD file descriptors needed to describe filespace selection*/ na_addr_t source = HG_Handler_get_addr(op_data->hg_handle); /* source address to pull data from */ @@ -741,7 +741,7 @@ H5VL_iod_server_attr_exists_cb(AXE_engine_t UNUSED axe_engine, const char *loc_name = input->path; /* path to start hierarchy traversal */ const char *attr_name = input->attr_name; /* attribute's name */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; iod_size_t kv_size = 0; htri_t ret = -1; herr_t ret_value = SUCCEED; @@ -853,7 +853,7 @@ H5VL_iod_server_attr_rename_cb(AXE_engine_t UNUSED axe_engine, iod_kv_t kv; /* KV entry */ H5VL_iod_link_t iod_link; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -970,7 +970,7 @@ H5VL_iod_server_attr_remove_cb(AXE_engine_t UNUSED axe_engine, iod_kv_t kv; H5VL_iod_link_t iod_link; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT diff --git a/src/H5VLiod_dset.c b/src/H5VLiod_dset.c index 377f5b7..a040f3a 100644 --- a/src/H5VLiod_dset.c +++ b/src/H5VLiod_dset.c @@ -151,7 +151,7 @@ H5VL_iod_server_dset_create_cb(AXE_engine_t UNUSED axe_engine, /* for the process that succeeded in creating the dataset, update the parent KV, create scratch pad */ if(0 == ret) { - uint32_t sp_cs; + iod_checksum_t sp_cs; /* create the attribute KV object for the dataset */ if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, @@ -290,7 +290,7 @@ H5VL_iod_server_dset_open_cb(AXE_engine_t UNUSED axe_engine, iod_obj_id_t dset_id; /* ID of the dataset to open */ iod_handle_t dset_oh, mdkv_oh; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -771,9 +771,20 @@ H5VL_iod_server_dset_get_vl_size_cb(AXE_engine_t UNUSED axe_engine, /* verify return values */ for(n=0 ; niod_oh, tid, NULL, - mem_desc, &file_desc, NULL, NULL) < 0) + mem_desc, &file_desc, &read_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read from array object"); + /* MSC - NEED IOD */ +#if 0 + /* compute checksum of blob ID and sequence length */ + cs.a = cs.b = cs.c = cs.state = 0; + cs.total_length = sizeof(iod_obj_id_t) + sizeof(iod_size_t); + entry_cs = H5checksum(&blob_id, sizeof(iod_obj_id_t), &cs); + entry_cs = H5checksum(&seq_len, sizeof(iod_size_t), &cs); + + if(entry_cs != read_cs) + HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data Corruption detected when reading"); +#endif + free(mem_desc); /* write operation */ @@ -1604,9 +1629,15 @@ H5VL__iod_server_vl_data_io_cb(void UNUSED *elem, hid_t type_id, unsigned ndims, mem_desc->frag[1].addr = &seq_len; mem_desc->frag[1].len = sizeof(iod_size_t); + /* compute checksum of blob ID and sequence length */ + cs.a = cs.b = cs.c = cs.state = 0; + cs.total_length = sizeof(iod_obj_id_t) + sizeof(iod_size_t); + entry_cs = H5checksum(&blob_id, sizeof(iod_obj_id_t), &cs); + entry_cs = H5checksum(&seq_len, sizeof(iod_size_t), &cs); + /* write the blob ID & size to the array element */ if(iod_array_write(udata->iod_oh, tid, NULL, - mem_desc, &file_desc, NULL, NULL) < 0) + mem_desc, &file_desc, &entry_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read from array object"); free(mem_desc); diff --git a/src/H5VLiod_dtype.c b/src/H5VLiod_dtype.c index 98c29e0..cb100fa 100644 --- a/src/H5VLiod_dtype.c +++ b/src/H5VLiod_dtype.c @@ -95,7 +95,8 @@ H5VL_iod_server_dtype_commit_cb(AXE_engine_t UNUSED axe_engine, /* for the process that succeeded in creating the datatype, update the parent KV, create scratch pad */ if(0 == ret) { - uint32_t sp_cs; + iod_checksum_t sp_cs; + iod_checksum_t dt_cs; /* create the metadata KV object for the datatype */ if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, @@ -145,8 +146,11 @@ H5VL_iod_server_dtype_commit_cb(AXE_engine_t UNUSED axe_engine, file_desc->frag[0].offset = 0; file_desc->frag[0].len = (iod_size_t)buf_size; + /* calculate a checksum for the datatype */ + dt_cs = H5checksum(buf, buf_size, NULL); + /* write the serialized type value to the BLOB object */ - if(iod_blob_write(dtype_oh, wtid, NULL, mem_desc, file_desc, NULL, NULL) < 0) + if(iod_blob_write(dtype_oh, wtid, NULL, mem_desc, file_desc, &dt_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to write BLOB object"); free(mem_desc); @@ -268,7 +272,8 @@ H5VL_iod_server_dtype_open_cb(AXE_engine_t UNUSED axe_engine, iod_mem_desc_t *mem_desc = NULL; /* memory descriptor used for reading */ iod_blob_iodesc_t *file_desc = NULL; /* file descriptor used to write */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; + iod_checksum_t dt_cs = 0, iod_cs = 0; iod_size_t kv_size; herr_t ret_value = SUCCEED; @@ -335,9 +340,19 @@ H5VL_iod_server_dtype_open_cb(AXE_engine_t UNUSED axe_engine, file_desc->frag[0].len = (iod_size_t)buf_size; /* read the serialized type value from the BLOB object */ - if(iod_blob_read(dtype_oh, rtid, NULL, &mem_desc, &file_desc, NULL, NULL) < 0) + if(iod_blob_read(dtype_oh, rtid, NULL, &mem_desc, &file_desc, &iod_cs, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to write BLOB object"); + /* MSC - NEED IOD */ +#if 0 + /* calculate a checksum for the datatype */ + dt_cs = H5checksum(buf, buf_size, NULL); + + /* Verifty checksum against one given by IOD */ + if(iod_cs != dt_cs) + HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data Corruption detected when reading datatype"); +#endif + free(mem_desc); free(file_desc); diff --git a/src/H5VLiod_file.c b/src/H5VLiod_file.c index ae58cd9..5be610bd 100644 --- a/src/H5VLiod_file.c +++ b/src/H5VLiod_file.c @@ -98,7 +98,7 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine, void *key = NULL; void *value = NULL; hid_t fcpl_id; - uint32_t sp_cs; + iod_checksum_t sp_cs; /* create the metadata KV object for the root group */ if(iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, @@ -240,7 +240,7 @@ H5VL_iod_server_file_open_cb(AXE_engine_t UNUSED axe_engine, iod_handle_t root_oh; /* root object handle */ iod_handle_t mdkv_oh; /* metadata object handle for KV to store file's metadata */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; iod_container_tids_t tids; iod_trans_id_t rtid; herr_t ret_value = SUCCEED; diff --git a/src/H5VLiod_group.c b/src/H5VLiod_group.c index 29cba23..9dca946 100644 --- a/src/H5VLiod_group.c +++ b/src/H5VLiod_group.c @@ -93,7 +93,7 @@ H5VL_iod_server_group_create_cb(AXE_engine_t UNUSED axe_engine, /* for the process that succeeded in creating the group, create the scratch pad for it too */ if(0 == ret) { - uint32_t sp_cs; + iod_checksum_t sp_cs; /* create the metadata KV object for the group */ if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, @@ -217,7 +217,7 @@ H5VL_iod_server_group_open_cb(AXE_engine_t UNUSED axe_engine, iod_obj_id_t grp_id; /* The ID of the group that needs to be opened */ iod_handle_t grp_oh, mdkv_oh; /* The group handle and its metadata KV handle */ scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT diff --git a/src/H5VLiod_link.c b/src/H5VLiod_link.c index b273d3b..25389f4 100644 --- a/src/H5VLiod_link.c +++ b/src/H5VLiod_link.c @@ -76,7 +76,7 @@ H5VL_iod_server_link_create_cb(AXE_engine_t UNUSED axe_engine, if(H5VL_LINK_CREATE_HARD == create_type) { scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; iod_handle_t mdkv_oh; uint64_t link_count = 0; @@ -269,7 +269,7 @@ H5VL_iod_server_link_move_cb(AXE_engine_t UNUSED axe_engine, iod_handle_t target_oh; iod_handle_t mdkv_oh; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; uint64_t link_count = 0; /* open the current group */ @@ -711,7 +711,7 @@ H5VL_iod_server_link_remove_cb(AXE_engine_t UNUSED axe_engine, iod_handle_t obj_oh; iod_handle_t mdkv_oh; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; uint64_t link_count = 0; obj_id = iod_link.u.iod_id; diff --git a/src/H5VLiod_map.c b/src/H5VLiod_map.c index 73ad1ae..7181553 100644 --- a/src/H5VLiod_map.c +++ b/src/H5VLiod_map.c @@ -98,7 +98,7 @@ H5VL_iod_server_map_create_cb(AXE_engine_t UNUSED axe_engine, /* for the process that succeeded in creating the map, create the scratch pad for it too */ if(0 == ret) { - uint32_t sp_cs; + iod_checksum_t sp_cs; /* create the metadata KV object for the map */ if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, @@ -217,7 +217,7 @@ H5VL_iod_server_map_open_cb(AXE_engine_t UNUSED axe_engine, iod_obj_id_t map_id; /* The ID of the map that needs to be opened */ iod_handle_t map_oh, mdkv_oh; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT diff --git a/src/H5VLiod_obj.c b/src/H5VLiod_obj.c index c126a56..19bf063 100644 --- a/src/H5VLiod_obj.c +++ b/src/H5VLiod_obj.c @@ -53,7 +53,7 @@ H5VL_iod_server_object_open_cb(AXE_engine_t UNUSED axe_engine, iod_obj_id_t obj_id; /* The ID of the object */ iod_handle_t mdkv_oh; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -265,7 +265,7 @@ H5VL_iod_server_object_copy_cb(AXE_engine_t UNUSED axe_engine, iod_size_t kv_size = sizeof(H5VL_iod_link_t); H5VL_iod_link_t iod_link; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -511,7 +511,7 @@ H5VL_iod_server_object_get_info_cb(AXE_engine_t UNUSED axe_engine, iod_handle_t obj_oh, mdkv_oh, attrkv_oh; iod_obj_id_t obj_id; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; H5I_type_t obj_type; iod_size_t num_attrs = 0; const char *loc_name = input->loc_name; @@ -646,7 +646,7 @@ H5VL_iod_server_object_set_comment_cb(AXE_engine_t UNUSED axe_engine, const char *loc_name = input->path; const char *comment = input->comment; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -743,7 +743,7 @@ H5VL_iod_server_object_get_comment_cb(AXE_engine_t UNUSED axe_engine, iod_obj_id_t obj_id; const char *loc_name = input->path; scratch_pad sp; - uint32_t sp_cs = 0; + iod_checksum_t sp_cs = 0; ssize_t size = 0; herr_t ret_value = SUCCEED; -- cgit v0.12