[svn-r24152] add more data integrity checks.

8 files changed, 77 insertions, 31 deletions

diff --git a/src/H5VLiod_attr.c b/src/H5VLiod_attr.c
index eab7abe..5c96b7e 100644
--- a/src/H5VLiod_attr.c
+++ b/src/H5VLiod_attr.c
@@ -60,7 +60,7 @@ H5VL_iod_server_attr_create_cb(AXE_engine_t UNUSED axe_engine,
     iod_array_struct_t array; /* IOD array structure for attribute's creation */
     iod_size_t *max_dims; /* MAX dims for IOD */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     iod_ret_t ret;
     hbool_t collective = FALSE; /* MSC - change when we allow for collective */
     herr_t ret_value = SUCCEED;
@@ -103,7 +103,7 @@ H5VL_iod_server_attr_create_cb(AXE_engine_t UNUSED axe_engine,
     /* for the process that succeeded in creating the attribute, update
        the parent scratch pad, create attribute scratch pad */
     if(0 == ret) {
-        uint32_t sp_cs;
+        iod_checksum_t sp_cs;
 
         /* create the metadata KV object for the attribute */
         if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, 
@@ -239,7 +239,7 @@ H5VL_iod_server_attr_open_cb(AXE_engine_t UNUSED axe_engine,
     const char *loc_name = input->path; /* current  path to start traversal */
     const char *attr_name = input->attr_name; /* attribute's name to open */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     H5VL_iod_link_t iod_link;
     herr_t ret_value = SUCCEED;
 
@@ -410,7 +410,7 @@ H5VL_iod_server_attr_read_cb(AXE_engine_t UNUSED axe_engine,
     void *buf; /* buffer to hold outgoing data */
     iod_handle_t mdkv_oh; /* metadata KV handle of attribute */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     int ndims; /* dataset's rank/number of dimensions */
     hssize_t num_descriptors = 0; /* number of IOD file descriptors needed to describe filespace selection */
     na_addr_t dest = HG_Handler_get_addr(op_data->hg_handle); /* destination address to push data to */
@@ -578,7 +578,7 @@ H5VL_iod_server_attr_write_cb(AXE_engine_t UNUSED axe_engine,
     void *buf; /* buffer to hold outgoing data */
     int ndims; /* dataset's rank/number of dimensions */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     iod_handle_t mdkv_oh; /* metadata KV handle of attribute */
     hssize_t num_descriptors = 0; /* number of IOD file descriptors needed to describe filespace selection*/
     na_addr_t source = HG_Handler_get_addr(op_data->hg_handle); /* source address to pull data from */
@@ -741,7 +741,7 @@ H5VL_iod_server_attr_exists_cb(AXE_engine_t UNUSED axe_engine,
     const char *loc_name = input->path; /* path to start hierarchy traversal */
     const char *attr_name = input->attr_name; /* attribute's name */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     iod_size_t kv_size = 0;
     htri_t ret = -1;
     herr_t ret_value = SUCCEED;
@@ -853,7 +853,7 @@ H5VL_iod_server_attr_rename_cb(AXE_engine_t UNUSED axe_engine,
     iod_kv_t kv; /* KV entry */
     H5VL_iod_link_t iod_link;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -970,7 +970,7 @@ H5VL_iod_server_attr_remove_cb(AXE_engine_t UNUSED axe_engine,
     iod_kv_t kv;
     H5VL_iod_link_t iod_link;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
diff --git a/src/H5VLiod_dset.c b/src/H5VLiod_dset.c
index 377f5b7..a040f3a 100644
--- a/src/H5VLiod_dset.c
+++ b/src/H5VLiod_dset.c
@@ -151,7 +151,7 @@ H5VL_iod_server_dset_create_cb(AXE_engine_t UNUSED axe_engine,
     /* for the process that succeeded in creating the dataset, update
        the parent KV, create scratch pad */
     if(0 == ret) {
-        uint32_t sp_cs;
+        iod_checksum_t sp_cs;
 
         /* create the attribute KV object for the dataset */
         if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, 
@@ -290,7 +290,7 @@ H5VL_iod_server_dset_open_cb(AXE_engine_t UNUSED axe_engine,
     iod_obj_id_t dset_id; /* ID of the dataset to open */
     iod_handle_t dset_oh, mdkv_oh;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -771,9 +771,20 @@ H5VL_iod_server_dset_get_vl_size_cb(AXE_engine_t UNUSED axe_engine,
 
     /* verify return values */
     for(n=0 ; n<num_descriptors ; n++) {
+        iod_checksum_t entry_cs = 0;
+
         if(ret_list[n] < 0)
             HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read from array object");
 
+        /* MSC - NEED IOD */
+#if 0
+        /* Verify checksum for that entry */
+        buf_ptr = (uint8_t *)buf;
+        entry_cs = H5checksum(buf_ptr, sizeof(iod_size_t) + sizeof(iod_obj_id_t), NULL);
+        if(entry_cs != io_array[n].cs)
+            HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data Corruption detected when reading");
+        buf_ptr += sizeof(iod_size_t) + sizeof(iod_obj_id_t);
+#endif
         free(io_array[n].mem_desc);
     }
 
@@ -1077,7 +1088,7 @@ H5VL_iod_server_dset_set_extent_cb(AXE_engine_t UNUSED axe_engine,
         int rank;
         hid_t space_id;
         scratch_pad sp;
-        uint32_t sp_cs = 0;
+        iod_checksum_t sp_cs = 0;
         iod_handle_t mdkv_oh;
 
         /* get scratch pad of the dataset */
@@ -1498,6 +1509,8 @@ H5VL__iod_server_vl_data_io_cb(void UNUSED *elem, hid_t type_id, unsigned ndims,
     iod_blob_iodesc_t *blob_desc; /* blob descriptor */
     size_t old_seq_len = 0;
     unsigned u;
+    iod_checksum_t entry_cs = 0, read_cs = 0;
+    H5_checksum_seed_t cs;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -1526,9 +1539,21 @@ H5VL__iod_server_vl_data_io_cb(void UNUSED *elem, hid_t type_id, unsigned ndims,
     file_desc = hslab;
 
     if(iod_array_read(udata->iod_oh, tid, NULL, 
-                      mem_desc, &file_desc, NULL, NULL) < 0)
+                      mem_desc, &file_desc, &read_cs, NULL) < 0)
         HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read from array object");
 
+    /* MSC - NEED IOD */
+#if 0
+    /* compute checksum of blob ID and sequence length */
+    cs.a = cs.b = cs.c = cs.state = 0;
+    cs.total_length = sizeof(iod_obj_id_t) + sizeof(iod_size_t);
+    entry_cs = H5checksum(&blob_id, sizeof(iod_obj_id_t), &cs);
+    entry_cs = H5checksum(&seq_len, sizeof(iod_size_t), &cs);
+
+    if(entry_cs != read_cs)
+        HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data Corruption detected when reading");
+#endif
+
     free(mem_desc);
 
     /* write operation */
@@ -1604,9 +1629,15 @@ H5VL__iod_server_vl_data_io_cb(void UNUSED *elem, hid_t type_id, unsigned ndims,
         mem_desc->frag[1].addr = &seq_len;
         mem_desc->frag[1].len = sizeof(iod_size_t);
 
+        /* compute checksum of blob ID and sequence length */
+        cs.a = cs.b = cs.c = cs.state = 0;
+        cs.total_length = sizeof(iod_obj_id_t) + sizeof(iod_size_t);
+        entry_cs = H5checksum(&blob_id, sizeof(iod_obj_id_t), &cs);
+        entry_cs = H5checksum(&seq_len, sizeof(iod_size_t), &cs);
+
         /* write the blob ID & size to the array element */
         if(iod_array_write(udata->iod_oh, tid, NULL, 
-                           mem_desc, &file_desc, NULL, NULL) < 0)
+                           mem_desc, &file_desc, &entry_cs, NULL) < 0)
             HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read from array object");
 
         free(mem_desc);
diff --git a/src/H5VLiod_dtype.c b/src/H5VLiod_dtype.c
index 98c29e0..cb100fa 100644
--- a/src/H5VLiod_dtype.c
+++ b/src/H5VLiod_dtype.c
@@ -95,7 +95,8 @@ H5VL_iod_server_dtype_commit_cb(AXE_engine_t UNUSED axe_engine,
     /* for the process that succeeded in creating the datatype, update
        the parent KV, create scratch pad */
     if(0 == ret) {
-        uint32_t sp_cs;
+        iod_checksum_t sp_cs;
+        iod_checksum_t dt_cs;
 
         /* create the metadata KV object for the datatype */
         if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, 
@@ -145,8 +146,11 @@ H5VL_iod_server_dtype_commit_cb(AXE_engine_t UNUSED axe_engine,
         file_desc->frag[0].offset = 0;
         file_desc->frag[0].len = (iod_size_t)buf_size;
 
+        /* calculate a checksum for the datatype */
+        dt_cs = H5checksum(buf, buf_size, NULL);
+
         /* write the serialized type value to the BLOB object */
-        if(iod_blob_write(dtype_oh, wtid, NULL, mem_desc, file_desc, NULL, NULL) < 0)
+        if(iod_blob_write(dtype_oh, wtid, NULL, mem_desc, file_desc, &dt_cs, NULL) < 0)
             HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to write BLOB object");
 
         free(mem_desc);
@@ -268,7 +272,8 @@ H5VL_iod_server_dtype_open_cb(AXE_engine_t UNUSED axe_engine,
     iod_mem_desc_t *mem_desc = NULL; /* memory descriptor used for reading */
     iod_blob_iodesc_t *file_desc = NULL; /* file descriptor used to write */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
+    iod_checksum_t dt_cs = 0, iod_cs = 0;
     iod_size_t kv_size;
     herr_t ret_value = SUCCEED;
 
@@ -335,9 +340,19 @@ H5VL_iod_server_dtype_open_cb(AXE_engine_t UNUSED axe_engine,
     file_desc->frag[0].len = (iod_size_t)buf_size;
 
     /* read the serialized type value from the BLOB object */
-    if(iod_blob_read(dtype_oh, rtid, NULL, &mem_desc, &file_desc, NULL, NULL) < 0)
+    if(iod_blob_read(dtype_oh, rtid, NULL, &mem_desc, &file_desc, &iod_cs, NULL) < 0)
         HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to write BLOB object");
 
+    /* MSC - NEED IOD */
+#if 0
+    /* calculate a checksum for the datatype */
+    dt_cs = H5checksum(buf, buf_size, NULL);
+
+    /* Verifty checksum against one given by IOD */
+    if(iod_cs != dt_cs)
+        HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "Data Corruption detected when reading datatype");
+#endif
+
     free(mem_desc);
     free(file_desc);
 
diff --git a/src/H5VLiod_file.c b/src/H5VLiod_file.c
index ae58cd9..5be610bd 100644
--- a/src/H5VLiod_file.c
+++ b/src/H5VLiod_file.c
@@ -98,7 +98,7 @@ H5VL_iod_server_file_create_cb(AXE_engine_t UNUSED axe_engine,
         void *key = NULL;
         void *value = NULL;
         hid_t fcpl_id;
-        uint32_t sp_cs;
+        iod_checksum_t sp_cs;
 
         /* create the metadata KV object for the root group */
         if(iod_obj_create(coh, first_tid, NULL, IOD_OBJ_KV, 
@@ -240,7 +240,7 @@ H5VL_iod_server_file_open_cb(AXE_engine_t UNUSED axe_engine,
     iod_handle_t root_oh; /* root object handle */
     iod_handle_t mdkv_oh; /* metadata object handle for KV to store file's metadata */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     iod_container_tids_t tids;
     iod_trans_id_t rtid;
     herr_t ret_value = SUCCEED;
diff --git a/src/H5VLiod_group.c b/src/H5VLiod_group.c
index 29cba23..9dca946 100644
--- a/src/H5VLiod_group.c
+++ b/src/H5VLiod_group.c
@@ -93,7 +93,7 @@ H5VL_iod_server_group_create_cb(AXE_engine_t UNUSED axe_engine,
     /* for the process that succeeded in creating the group, create
        the scratch pad for it too */
     if(0 == ret) {
-        uint32_t sp_cs;
+        iod_checksum_t sp_cs;
 
         /* create the metadata KV object for the group */
         if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, 
@@ -217,7 +217,7 @@ H5VL_iod_server_group_open_cb(AXE_engine_t UNUSED axe_engine,
     iod_obj_id_t grp_id; /* The ID of the group that needs to be opened */
     iod_handle_t grp_oh, mdkv_oh; /* The group handle and its metadata KV handle */
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
diff --git a/src/H5VLiod_link.c b/src/H5VLiod_link.c
index b273d3b..25389f4 100644
--- a/src/H5VLiod_link.c
+++ b/src/H5VLiod_link.c
@@ -76,7 +76,7 @@ H5VL_iod_server_link_create_cb(AXE_engine_t UNUSED axe_engine,
 
     if(H5VL_LINK_CREATE_HARD == create_type) {
         scratch_pad sp;
-        uint32_t sp_cs = 0;
+        iod_checksum_t sp_cs = 0;
         iod_handle_t mdkv_oh;
         uint64_t link_count = 0;
 
@@ -269,7 +269,7 @@ H5VL_iod_server_link_move_cb(AXE_engine_t UNUSED axe_engine,
         iod_handle_t target_oh;
         iod_handle_t mdkv_oh;
         scratch_pad sp;
-        uint32_t sp_cs = 0;
+        iod_checksum_t sp_cs = 0;
         uint64_t link_count = 0;
 
         /* open the current group */
@@ -711,7 +711,7 @@ H5VL_iod_server_link_remove_cb(AXE_engine_t UNUSED axe_engine,
         iod_handle_t obj_oh;
         iod_handle_t mdkv_oh;
         scratch_pad sp;
-        uint32_t sp_cs = 0;
+        iod_checksum_t sp_cs = 0;
         uint64_t link_count = 0;
 
         obj_id = iod_link.u.iod_id;
diff --git a/src/H5VLiod_map.c b/src/H5VLiod_map.c
index 73ad1ae..7181553 100644
--- a/src/H5VLiod_map.c
+++ b/src/H5VLiod_map.c
@@ -98,7 +98,7 @@ H5VL_iod_server_map_create_cb(AXE_engine_t UNUSED axe_engine,
     /* for the process that succeeded in creating the map, create
        the scratch pad for it too */
     if(0 == ret) {
-        uint32_t sp_cs;
+        iod_checksum_t sp_cs;
 
         /* create the metadata KV object for the map */
         if(iod_obj_create(coh, wtid, NULL, IOD_OBJ_KV, 
@@ -217,7 +217,7 @@ H5VL_iod_server_map_open_cb(AXE_engine_t UNUSED axe_engine,
     iod_obj_id_t map_id; /* The ID of the map that needs to be opened */
     iod_handle_t map_oh, mdkv_oh;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
diff --git a/src/H5VLiod_obj.c b/src/H5VLiod_obj.c
index c126a56..19bf063 100644
--- a/src/H5VLiod_obj.c
+++ b/src/H5VLiod_obj.c
@@ -53,7 +53,7 @@ H5VL_iod_server_object_open_cb(AXE_engine_t UNUSED axe_engine,
     iod_obj_id_t obj_id; /* The ID of the object */
     iod_handle_t mdkv_oh;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -265,7 +265,7 @@ H5VL_iod_server_object_copy_cb(AXE_engine_t UNUSED axe_engine,
     iod_size_t kv_size = sizeof(H5VL_iod_link_t);
     H5VL_iod_link_t iod_link;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -511,7 +511,7 @@ H5VL_iod_server_object_get_info_cb(AXE_engine_t UNUSED axe_engine,
     iod_handle_t obj_oh, mdkv_oh, attrkv_oh;
     iod_obj_id_t obj_id;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     H5I_type_t obj_type;
     iod_size_t num_attrs = 0;
     const char *loc_name = input->loc_name;
@@ -646,7 +646,7 @@ H5VL_iod_server_object_set_comment_cb(AXE_engine_t UNUSED axe_engine,
     const char *loc_name = input->path;
     const char *comment = input->comment;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -743,7 +743,7 @@ H5VL_iod_server_object_get_comment_cb(AXE_engine_t UNUSED axe_engine,
     iod_obj_id_t obj_id;
     const char *loc_name = input->path;
     scratch_pad sp;
-    uint32_t sp_cs = 0;
+    iod_checksum_t sp_cs = 0;
     ssize_t size = 0;
     herr_t ret_value = SUCCEED;