From 373038013ea82651f9fe587a0439cab710ede0b8 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 13 Jun 2013 19:30:20 -0500 Subject: [svn-r23769] add property to indicate append add a new version of checksum algorithm to correctly save the state other updates... --- examples/test_client.c | 203 ++++++++++++++++++++++++++++++++++++------------- src/H5Pdxpl.c | 16 ++++ src/H5Sprivate.h | 2 + src/H5Sselect.c | 75 ++++++++++++++++++ src/H5VLiod.c | 53 ++++++++++++- src/H5VLiod.h | 3 + src/H5VLiod_client.c | 14 ++-- src/H5VLiod_client.h | 4 +- src/H5VLiod_server.c | 19 ++++- src/H5checksum.c | 169 ++++++++++++++++++++++++++++++++++++++++ src/H5private.h | 1 + src/H5public.h | 9 +++ 12 files changed, 500 insertions(+), 68 deletions(-) diff --git a/examples/test_client.c b/examples/test_client.c index a23188f..c066e81 100644 --- a/examples/test_client.c +++ b/examples/test_client.c @@ -34,6 +34,7 @@ int main(int argc, char **argv) { int num_requests = 0; hsize_t extent = 20; hbool_t exists; + herr_t ret; uint32_t cs; H5_request_t req1; H5_status_t status1; @@ -55,6 +56,7 @@ int main(int argc, char **argv) { MPI_Comm_size(MPI_COMM_WORLD, &my_size); fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank); + fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n"); /* Choose the IOD VOL plugin to use with this file. First we create a file access property list. Then we call a new routine to set the IOD plugin to use with this fapl */ @@ -97,12 +99,11 @@ int main(int argc, char **argv) { /* create the file. This is asynchronous, but the file_id can be used. */ file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, event_q); + assert(file_id); - { - char temp_name[50]; - H5Fget_name(file_id, temp_name, 50); - fprintf(stderr, "File name %s %s\n", temp_name, file_name); - } + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Create Three Groups, a Named datatype, an Attribute, and 3 Datasets \n"); + fprintf(stderr, "*****************************************************************************************************************\n"); /* create 3 groups in the file. All calls are completely asychronous. Even though they depend one each other here and @@ -111,7 +112,8 @@ int main(int argc, char **argv) { server, dependencies are noted in the AXE and functions execute in the order they are supposed to execute. */ gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q); - H5Oset_comment_ff(gid1, "Testing Object Comment", 0, event_q); + ret = H5Oset_comment_ff(gid1, "Testing Object Comment", 0, event_q); + assert(ret == 0); gid2 = H5Gcreate_ff(file_id, "G1/G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q); gid3 = H5Gcreate_ff(file_id, "G1/G2/G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q); @@ -127,8 +129,9 @@ int main(int argc, char **argv) { * Other Local H5T type operations can be issued before completing this call * because they do not depend on the committed state of the datatype. */ - H5Tcommit_ff(file_id, "int", int_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, - 0, event_q); + ret = H5Tcommit_ff(file_id, "int", int_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, + 0, event_q); + assert(ret == 0); /* create a dataspace. This is a local Bookeeping operation that does not touch the file */ @@ -146,12 +149,14 @@ int main(int argc, char **argv) { asynchronous. The attribute id can not be used anymore, however the creation operation will occur first at the server before the close */ - H5Aclose_ff(aid1, event_q); + ret = H5Aclose_ff(aid1, event_q); + assert(ret == 0); /* Check if the attribute on group G1 exists. This is asynchronous, so the exists status is correct to examine after the completion of the operation */ - assert(H5Aexists_by_name_ff(file_id,"G1","ATTR1", H5P_DEFAULT, &exists, 0, event_q) == 0); + ret = H5Aexists_by_name_ff(file_id,"G1","ATTR1", H5P_DEFAULT, &exists, 0, event_q); + assert(ret == 0); /* Pop the request for the last operation from the event queue. The Event queue does not manage it anymore and it is the @@ -169,11 +174,13 @@ int main(int argc, char **argv) { printf("Attribute ATTR1 does NOT exist. This must be the test without a native backend\n"); /* Delete the attribute just created, this is asynchronous */ - assert(H5Adelete_by_name_ff(file_id, "G1", "ATTR1", H5P_DEFAULT, 0, event_q) == 0); + ret = H5Adelete_by_name_ff(file_id, "G1", "ATTR1", H5P_DEFAULT, 0, event_q); + assert(ret == 0); /* check if it exists now. This is asynchronous. Can't check the exists status until the operation completes. */ - assert(0 == H5Aexists_ff(gid1, "ATTR1", &exists, 0, event_q)); + ret = H5Aexists_ff(gid1, "ATTR1", &exists, 0, event_q); + assert(ret == 0); /* create a Dataset D1 on the file, in group /G1/G2/G3. This is asynchronous. Internally to the IOD-VOL this call traverses @@ -191,11 +198,13 @@ int main(int argc, char **argv) { /* rename the attribute. This is asynchronous with a dependency on the create at the server*/ - H5Arename_ff(did1, "ATTR2_tmp", "ATTR2", 0, event_q); + ret = H5Arename_ff(did1, "ATTR2_tmp", "ATTR2", 0, event_q); + assert(ret == 0); /* write data to the attribute. This is asynchronous but will wait for the creation and rename calls at the server to complete */ - H5Awrite_ff(aid2, int_id, a_data, 0, event_q); + ret = H5Awrite_ff(aid2, int_id, a_data, 0, event_q); + assert(ret == 0); /* similar to the previous H5Dcreate. */ did2 = H5Dcreate_ff(file_id,"G1/G2/G3/D2",int_id,dataspaceId, @@ -207,10 +216,14 @@ int main(int argc, char **argv) { H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, 0, event_q); assert(did3); + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Perform Dataset I/O operations on the created datasets\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* Attach a checksum to the dxpl which is verified all the way down at the server */ dxpl_id = H5Pcreate (H5P_DATASET_XFER); - cs = H5_checksum_fletcher32(data, sizeof(int) * nelem); + cs = H5_checksum_lookup3(data, sizeof(int) * nelem, 0); H5Pset_dxpl_checksum(dxpl_id, cs); /* Raw data write on D1. This is asynchronous, but it is delayed @@ -218,27 +231,28 @@ int main(int argc, char **argv) { is completed. Internal to the IOD-VOL plugin client we generate a checksum for data and ship it with the write bulk data function shipper handle to the server. */ - H5Dwrite_ff(did1, int_id, dataspaceId, dataspaceId, dxpl_id, data, - 0, event_q); - - cs = H5_checksum_fletcher32(data2, sizeof(int) * nelem); + ret = H5Dwrite_ff(did1, int_id, dataspaceId, dataspaceId, dxpl_id, data, + 0, event_q); + assert(ret == 0); + + /* Raw data write on D2. same as previous, but here we indicate + through the property list that we want to inject a + corruption. */ + cs = H5_checksum_lookup3(data2, sizeof(int) * nelem, 0); H5Pset_dxpl_checksum(dxpl_id, cs); - - /* Raw data write on D2. same as previous. */ - H5Dwrite_ff(did2, int_id, dataspaceId, dataspaceId, dxpl_id, data2, - 0, event_q); - + H5Pset_dxpl_inject_bad_checksum(dxpl_id, 1); + ret = H5Dwrite_ff(did2, int_id, dataspaceId, dataspaceId, dxpl_id, data2, + 0, event_q); + assert(ret == 0); H5Pclose(dxpl_id); /* Raw data write on D3. Same as previous; however we specify that - the data in the buffer is in BE byte order and of smaller - extent than the datatype of the dataset. Type conversion will + the data in the buffer is in BE byte order. Type conversion will happen at the server when we detect that the dataset type is of - LE order and the datatype here is in BE order and the - difference in size. The buffer is resized at the server also to - encompass the increase. */ - H5Dwrite_ff(did3, H5T_STD_I16LE, dataspaceId, dataspaceId, H5P_DEFAULT, data3, - 0, event_q); + LE order and the datatype here is in BE order. */ + ret = H5Dwrite_ff(did3, H5T_STD_I32BE, dataspaceId, dataspaceId, H5P_DEFAULT, data3, + 0, event_q); + assert(ret == 0); /* NOTE: all raw data reads/writes execute concurrently at the server if they get scheduled by the Asynchronous eXecution @@ -267,8 +281,9 @@ int main(int argc, char **argv) { The server returns, along with the data array, a checksum for the data that should be stored, but for now generated. */ - H5Dread_ff(did1, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, r_data, - 0, event_q); + ret = H5Dread_ff(did1, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, r_data, + 0, event_q); + assert(ret == 0); /* Pop head request from the event queue to test it next. This is the request that belongs to the previous H5Dread_ff call. */ @@ -296,8 +311,9 @@ int main(int argc, char **argv) { fail the close, but just print a Fatal error. */ dxpl_id = H5Pcreate (H5P_DATASET_XFER); H5Pset_dxpl_inject_bad_checksum(dxpl_id, 1); - H5Dread_ff(did1, int_id, dataspaceId, dataspaceId, dxpl_id, r2_data, - 0, event_q); + ret = H5Dread_ff(did1, int_id, dataspaceId, dataspaceId, dxpl_id, r2_data, + 0, event_q); + assert(ret == 0); H5Pclose(dxpl_id); /* Issue other operations to query certain metadata values or @@ -307,7 +323,9 @@ int main(int argc, char **argv) { char temp_name[50]; hid_t plist_id; - H5Fget_intent(file_id, &intent); + ret = H5Fget_intent(file_id, &intent); + assert(ret == 0); + fprintf(stderr, "Intent %d %d\n", intent, H5F_ACC_RDWR); H5Fget_name(gid1, temp_name, 50); fprintf(stderr, "File name %s %s\n", temp_name, file_name); @@ -321,9 +339,14 @@ int main(int argc, char **argv) { H5Pclose(plist_id); /* change the dataset dimensions for Dataset D1. */ - assert(H5Dset_extent_ff(did1, &extent, event_q) == 0); + ret = H5Dset_extent_ff(did1, &extent, event_q); + assert(ret == 0); } + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Close Objects\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* closing did1 acts as a barrier task at the server for all operations dependeing on the dataset. This is asynchronous. */ assert(H5Dclose_ff(did1, event_q) == 0); @@ -336,32 +359,47 @@ int main(int argc, char **argv) { assert(H5Gclose_ff(gid2, event_q) == 0); assert(H5Gclose_ff(gid3, event_q) == 0); - /* Test Links */ + + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Test Links\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); /* create two new groups /G4 and /G4/G5 */ gid1 = H5Gcreate_ff(file_id, "G4", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q); + assert(gid1); gid2 = H5Gcreate_ff(file_id, "G4/G5", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0, event_q); + assert(gid2); /* create a hard link to D1 (created previosuly) so that it can be accessed from G5/newD1. This is asynchronous; however all operation that depend on G5 now will have this operation as a parent at the AXE on the server. */ - H5Lcreate_hard_ff(file_id, "G1/G2/G3/D1", gid1, "G5/newD1", H5P_DEFAULT, H5P_DEFAULT, 0, event_q); + ret = H5Lcreate_hard_ff(file_id, "G1/G2/G3/D1", gid1, "G5/newD1", H5P_DEFAULT, H5P_DEFAULT, 0, event_q); + assert(ret == 0); /* Try and open the dataset. This is asynchronous. */ did1 = H5Dopen_ff(file_id,"G4/G5/newD1", H5P_DEFAULT, 0, event_q); + assert(did1); - H5Lcreate_soft_ff("/G1/G2/G3/D4", gid1, "G5/newD2", H5P_DEFAULT, H5P_DEFAULT, 0, event_q); - H5Lmove_ff(file_id, "/G1/G2/G3/D3", file_id, "/G4/G5/D3moved", H5P_DEFAULT, H5P_DEFAULT, 0, event_q); - H5Ldelete_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, 0, event_q); + ret = H5Lcreate_soft_ff("/G1/G2/G3/D4", gid1, "G5/newD2", H5P_DEFAULT, H5P_DEFAULT, 0, event_q); + assert(ret == 0); + ret = H5Lmove_ff(file_id, "/G1/G2/G3/D3", file_id, "/G4/G5/D3moved", H5P_DEFAULT, H5P_DEFAULT, 0, event_q); + assert(ret == 0); + ret = H5Ldelete_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, 0, event_q); + assert(ret == 0); assert(H5Dclose_ff(did1, event_q) == 0); assert(H5Gclose_ff(gid1, event_q) == 0); assert(H5Gclose_ff(gid2, event_q) == 0); + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Flush and Close File\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* flush all the contents of file to disk. This is a barrier task at the server. This is asynchronous. */ - assert(H5Fflush_ff(file_id, H5F_SCOPE_GLOBAL, event_q) == 0); + ret = H5Fflush_ff(file_id, H5F_SCOPE_GLOBAL, event_q); + assert(ret == 0); /* If the read request did no complete earlier when we tested it, Wait on it now. We have to do this since we popped it earlier from the event queue */ @@ -376,6 +414,10 @@ int main(int argc, char **argv) { on the container. */ assert(H5Fclose_ff(file_id, event_q) == 0); + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Wait on everything in EQ and check Results of operations in EQ\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* wait on all requests and print completion status */ H5EQwait(event_q, &num_requests, &status); fprintf(stderr, "%d requests in event queue. Completions: ", num_requests); @@ -414,6 +456,11 @@ int main(int argc, char **argv) { fprintf(stderr, "%d ",r2_data[i]); fprintf(stderr, "\n"); + + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Open The Container\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* Now we test the Open routines. Since there is no underneath container, the underlying VOL server is just going to "fake" open calls and assume they exist. However there is no metadata @@ -427,13 +474,19 @@ int main(int argc, char **argv) { file_id = H5Fopen_ff(file_name, H5F_ACC_RDONLY, fapl_id, event_q); assert(file_id); + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Test H5O* routines. H5Oopens are synchronous\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* test object copy. This is asynchronous */ - assert(H5Ocopy_ff(file_id, "/G1/G2/G3/D1", file_id, "D1_copy", - H5P_DEFAULT, H5P_DEFAULT, 0, event_q) == 0); + ret = H5Ocopy_ff(file_id, "/G1/G2/G3/D1", file_id, "D1_copy", + H5P_DEFAULT, H5P_DEFAULT, 0, event_q); + assert(ret == 0); /* check if an object exists. This is asynchronous, so checking the value should be done after the wait */ - H5Oexists_by_name_ff(file_id, "G1/G2/G3", &exists, H5P_DEFAULT, 0, event_q); + ret = H5Oexists_by_name_ff(file_id, "G1/G2/G3", &exists, H5P_DEFAULT, 0, event_q); + assert(ret == 0); /* Open objects using the general H5O routines. This has an asynchronous interface, but right now we implement it @@ -456,6 +509,10 @@ int main(int argc, char **argv) { assert(H5Oclose_ff(int_id, event_q) == 0); assert(H5Oclose_ff(gid1, event_q) == 0); + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Test Regular object Open Containers and Read from them\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* Open a group G1 on the file. This is asynchronous */ gid1 = H5Gopen_ff(file_id, "G1", H5P_DEFAULT, 0, event_q); assert(gid1); @@ -465,7 +522,9 @@ int main(int argc, char **argv) { ssize_t ret_size; char *comment = NULL; - H5Oget_comment_ff(gid1, NULL, 0, &ret_size, 0, event_q); + ret = H5Oget_comment_ff(gid1, NULL, 0, &ret_size, 0, event_q); + assert(ret == 0); + if(H5EQpop(event_q, &req1) < 0) exit(1); assert(H5AOwait(req1, &status1) == 0); @@ -474,7 +533,8 @@ int main(int argc, char **argv) { comment = malloc((size_t)ret_size); - H5Oget_comment_ff(gid1, comment, (size_t)ret_size + 1, &ret_size, 0, event_q); + ret = H5Oget_comment_ff(gid1, comment, (size_t)ret_size + 1, &ret_size, 0, event_q); + assert(ret == 0); if(H5EQpop(event_q, &req1) < 0) exit(1); assert(H5AOwait(req1, &status1) == 0); @@ -503,8 +563,9 @@ int main(int argc, char **argv) { client. Printing this data will result in 0 - 60 in 16 bit BE byte order. */ memset(r_data, 0, nelem*sizeof(int)); - H5Dread_ff(did1, H5T_STD_I16BE, dataspaceId, dataspaceId, H5P_DEFAULT, r_data, - 0, event_q); + ret = H5Dread_ff(did1, H5T_STD_I16BE, dataspaceId, dataspaceId, H5P_DEFAULT, r_data, + 0, event_q); + assert(ret == 0); if(H5EQpop(event_q, &req1) < 0) exit(1); assert(H5AOwait(req1, &status1) == 0); @@ -532,14 +593,39 @@ int main(int argc, char **argv) { mem_space = H5Screate_simple(1, dims, NULL); H5Sselect_hyperslab(mem_space, H5S_SELECT_SET, &start,&stride,&count,&block); - H5Dread_ff(did1, H5T_STD_I32LE, mem_space, dataspaceId, H5P_DEFAULT, buf, - 0, event_q); + ret = H5Dread_ff(did1, H5T_STD_I32LE, mem_space, dataspaceId, H5P_DEFAULT, buf, + 0, event_q); + assert(ret == 0); if(H5EQpop(event_q, &req1) < 0) exit(1); assert(H5AOwait(req1, &status1) == 0); assert (status1); +#if 1 + { + int i; + uint32_t temp = 0, temp1 = 0, temp2 = 0; + int buf1[60], buf2[120]; + H5_checksum_seed_t cs; + + cs.a = cs.b = cs.c = cs.state = 0; + cs.total_length = 60*4; + + for(i=0 ; i<60 ; i++) { + buf1[i] = i; + buf2[i*2] = i; + } + + for(i=0 ; i<16 ; i++) { + temp1 = H5_checksum_lookup4(&((char *)buf1)[i*15], 15, &cs); + printf("Current state = %u\n", cs.state); + } + temp = H5_checksum_lookup4(&((char *)buf1)[0], 240, NULL); + printf("Example cs = %u %u\n", temp, temp1); + } +#endif + fprintf(stderr, "Printing all Dataset values. We should have a 0 after each element: "); for(i=0;i<120;++i) fprintf(stderr, "%d ", buf[i]); @@ -556,9 +642,14 @@ int main(int argc, char **argv) { assert(aid2); /* read data from attribute. this is asynchronous */ - H5Aread_ff(aid2, int_id, ra_data, 0, event_q); - assert(H5Aclose(aid2) == 0); + ret = H5Aread_ff(aid2, int_id, ra_data, 0, event_q); + assert(ret == 0); + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Close all open objects then Wait for events in EQ\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + + assert(H5Aclose(aid2) == 0); assert(H5Tclose(int_id) == 0); assert(H5Gclose(gid1) == 0); assert(H5Fclose(file_id) == 0); @@ -602,6 +693,10 @@ int main(int argc, char **argv) { free(r3_data); free(data3); + fprintf(stderr, "\n*****************************************************************************************************************\n"); + fprintf(stderr, "Finalize EFF stack\n"); + fprintf(stderr, "*****************************************************************************************************************\n"); + /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server and shutsdown the FS server (when all clients send the terminate request) and client */ diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index e5578f9..146ae96 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -48,6 +48,7 @@ /****************/ #ifdef H5_HAVE_EFF + #define H5D_XFER_INJECT_BAD_CHECKSUM_SIZE sizeof(hbool_t) #define H5D_XFER_INJECT_BAD_CHECKSUM_DEF FALSE #define H5D_XFER_INJECT_BAD_CHECKSUM_ENC H5P__encode_hbool_t @@ -57,6 +58,12 @@ #define H5D_XFER_CHECKSUM_DEF 0 #define H5D_XFER_CHECKSUM_ENC H5P__encode_unsigned #define H5D_XFER_CHECKSUM_DEC H5P__decode_unsigned + +#define H5D_XFER_APPEND_ONLY_SIZE sizeof(hbool_t) +#define H5D_XFER_APPEND_ONLY_DEF FALSE +#define H5D_XFER_APPEND_ONLY_ENC H5P__encode_hbool_t +#define H5D_XFER_APPEND_ONLY_DEC H5P__decode_hbool_t + #endif /* H5_HAVE_EFF */ /* ======== Data transfer properties ======== */ @@ -252,6 +259,7 @@ const H5P_libclass_t H5P_CLS_DXFR[1] = {{ #ifdef H5_HAVE_EFF static const hbool_t H5D_def_inject_bad_checksum_g = H5D_XFER_INJECT_BAD_CHECKSUM_DEF; static const uint32_t H5D_def_checksum_g = H5D_XFER_CHECKSUM_DEF; +static const hbool_t H5D_def_append_only_g = H5D_XFER_APPEND_ONLY_DEF; #endif /* H5_HAVE_EFF */ /* Property value defaults */ @@ -307,6 +315,7 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) FUNC_ENTER_STATIC #ifdef H5_HAVE_EFF + if(H5P_register_real(pclass, H5D_XFER_INJECT_BAD_CHECKSUM_NAME, H5D_XFER_INJECT_BAD_CHECKSUM_SIZE, &H5D_def_inject_bad_checksum_g, NULL, NULL, NULL, H5D_XFER_INJECT_BAD_CHECKSUM_ENC, H5D_XFER_INJECT_BAD_CHECKSUM_DEC, @@ -318,6 +327,13 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, H5D_XFER_CHECKSUM_ENC, H5D_XFER_CHECKSUM_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + if(H5P_register_real(pclass, H5D_XFER_APPEND_ONLY_NAME, H5D_XFER_APPEND_ONLY_SIZE, + &H5D_def_append_only_g, + NULL, NULL, NULL, H5D_XFER_APPEND_ONLY_ENC, H5D_XFER_APPEND_ONLY_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + #endif /* H5_HAVE_EFF */ /* Register the max. temp buffer size property */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 63b31bc..34fc07f 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -272,6 +272,8 @@ H5_DLL H5S_t *H5S_decode(const unsigned char *buf); H5_DLL herr_t H5S_get_offsets(const H5S_t *space, size_t elmt_size, size_t nelmts, hsize_t **_off, size_t **_len, size_t *_num_entries); +H5_DLL uint32_t H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, + const H5S_t *space); #ifdef H5_HAVE_PARALLEL /* Global vars whose value comes from environment variable */ diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 23f87b1..1d6c63d 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -2143,3 +2143,78 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_get_offsets() */ + + +/*------------------------------------------------------------------------- + * Function: H5S_checksum + * + * Purpose: Computes a checksum for a buffer with a dataspace + * selection using the HDF5 fletcher checksume routines. + * + * Return: Returns a 32-bit value. Every bit of the key affects every bit of + * the return value. Two keys differing by one or two bits will have + * totally different hash values. + * + * Programmer: Mohamad Chaarawi + * June 2013 + * + *------------------------------------------------------------------------- + */ +uint32_t +H5S_checksum(const void *buf, size_t elmt_size, size_t nelmts, const H5S_t *space) +{ + hsize_t _off[H5D_IO_VECTOR_SIZE]; /* Array to store sequence offsets in memory */ + hsize_t *off = NULL; /* Pointer to sequence offsets in memory */ + size_t _len[H5D_IO_VECTOR_SIZE]; /* Array to store sequence lengths in memory */ + size_t *len = NULL; /* Pointer to sequence lengths in memory */ + H5S_sel_iter_t iter; /* Memory selection iteration info */ + hbool_t iter_init = 0; /* Memory selection iteration info has been initialized */ + size_t nseq; /* Number of sequences generated */ + uint32_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT + + len = _len; + off = _off; + + /* Check for only one element in selection */ + if(nelmts == 1) { + size_t buf_size = elmt_size * nelmts; + + if(H5S_SELECT_OFFSET(space, off) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, 0, "can't retrieve memory selection offset"); + + ret_value = H5_checksum_lookup3(buf, buf_size, ret_value); + } /* end if */ + else { + size_t nelem; /* Number of elements used in sequences */ + size_t i; + + /* Initialize iterator */ + if(H5S_select_iter_init(&iter, space, elmt_size) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, 0, "unable to initialize selection iterator") + iter_init = 1; + + nseq = 0; + + /* Loop, until all bytes are processed */ + while(nelmts > 0) { + /* Get sequences for selection */ + if(H5S_SELECT_GET_SEQ_LIST(space, 0, &iter, H5D_IO_VECTOR_SIZE, nelmts, &nseq, + &nelem, off, len) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); + + for(i=0 ; istatus = status; info->bulk_handle = bulk_handle; info->buf_ptr = buf; - info->buf_size = buf_size; + info->nelmts = nelmts; + info->type_size = type_size; + if(NULL == (info->space = H5S_copy(mem_space, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to copy dataspace") /* Get async request for operation */ if(do_async) { @@ -2451,7 +2496,7 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id, buf_size = nelmts * type_size; /* calculate a checksum for the data */ - internal_cs = H5_checksum_fletcher32(buf, buf_size); + internal_cs = H5S_checksum(buf, type_size, nelmts, mem_space); /* Verify the checksum value if the dxpl contains a user defined checksum */ if(H5P_DATASET_XFER_DEFAULT != dxpl_id) { @@ -2552,7 +2597,7 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id, /* setup info struct for I/O request This is to manage the I/O operation once the wait is called. */ - if(NULL == (info = (H5VL_iod_io_info_t *)H5MM_malloc(sizeof(H5VL_iod_io_info_t)))) + if(NULL == (info = (H5VL_iod_io_info_t *)H5MM_calloc(sizeof(H5VL_iod_io_info_t)))) HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "can't allocate a request"); info->status = status; info->bulk_handle = bulk_handle; diff --git a/src/H5VLiod.h b/src/H5VLiod.h index 773f51b..a813f59 100644 --- a/src/H5VLiod.h +++ b/src/H5VLiod.h @@ -51,6 +51,7 @@ extern "C" { #define H5D_XFER_INJECT_BAD_CHECKSUM_NAME "inject_bad_checksum" #define H5D_XFER_CHECKSUM_NAME "checksum" +#define H5D_XFER_APPEND_ONLY_NAME "append_only" H5_DLL H5VL_class_t *H5VL_iod_init(void); H5_DLL herr_t H5Pset_fapl_iod(hid_t fapl_id, MPI_Comm comm, MPI_Info info); @@ -61,6 +62,8 @@ H5_DLL herr_t H5Pset_dxpl_checksum(hid_t dxpl_id, uint32_t flag); H5_DLL herr_t H5Pget_dxpl_checksum(hid_t dxpl_id, uint32_t *flag); H5_DLL herr_t H5Pset_dxpl_inject_bad_checksum(hid_t dxpl_id, hbool_t flag); H5_DLL herr_t H5Pget_dxpl_inject_bad_checksum(hid_t dxpl_id, hbool_t *flag); +H5_DLL herr_t H5Pset_dxpl_append_only(hid_t dxpl_id, hbool_t flag); +H5_DLL herr_t H5Pget_dxpl_append_only(hid_t dxpl_id, hbool_t *flag); #endif /* H5_HAVE_EFF */ #ifdef __cplusplus diff --git a/src/H5VLiod_client.c b/src/H5VLiod_client.c index 022f5ab..a34521c 100644 --- a/src/H5VLiod_client.c +++ b/src/H5VLiod_client.c @@ -26,6 +26,7 @@ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5Sprivate.h" /* Dataspaces */ #include "H5VLprivate.h" /* VOL plugins */ #include "H5VLiod.h" /* Iod VOL plugin */ #include "H5VLiod_common.h" @@ -346,7 +347,7 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) req->state = H5VL_IOD_COMPLETED; } if(HG_DSET_WRITE == req->type && SUCCEED != *((int *)info->status)) { - fprintf(stderr, "write failed %d\n", *((int *)info->status)); + fprintf(stderr, "Errrr! Dataset Write Failure Reported from Server\n"); req->status = H5AO_FAILED; req->state = H5VL_IOD_COMPLETED; } @@ -354,7 +355,7 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) H5VL_iod_read_status_t *read_status = (H5VL_iod_read_status_t *)info->status; if(SUCCEED != read_status->ret) { - fprintf(stderr, "read failed\n"); + fprintf(stderr, "Errrr! Dataset Read Failure Reported from Server\n"); req->status = H5AO_FAILED; req->state = H5VL_IOD_COMPLETED; } @@ -362,15 +363,18 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) uint32_t internal_cs; /* calculate a checksum for the data recieved */ - internal_cs = H5_checksum_fletcher32(info->buf_ptr, info->buf_size); + internal_cs = H5S_checksum(info->buf_ptr, info->type_size, + info->nelmts, info->space); /* verify data integrity */ if(internal_cs != read_status->cs) { - fprintf(stderr, "Errrrr! Data integrity failure (expecting %u got %u).\n", + fprintf(stderr, "Errrrr! Dataset Read integrity failure (expecting %u got %u).\n", read_status->cs, internal_cs); req->status = H5AO_FAILED; req->state = H5VL_IOD_COMPLETED; } + if(info->space && H5S_close(info->space) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release dataspace") } } @@ -394,7 +398,7 @@ H5VL_iod_request_complete(H5VL_iod_file_t *file, H5VL_iod_request_t *req) req->state = H5VL_IOD_COMPLETED; } if(SUCCEED != *((int *)info->status)) { - fprintf(stderr, "write failed %d\n", *((int *)info->status)); + fprintf(stderr, "Attribute I/O Failure Reported from Server\n"); req->status = H5AO_FAILED; req->state = H5VL_IOD_COMPLETED; } diff --git a/src/H5VLiod_client.h b/src/H5VLiod_client.h index 1722690..9e10328 100644 --- a/src/H5VLiod_client.h +++ b/src/H5VLiod_client.h @@ -196,7 +196,9 @@ typedef struct H5VL_iod_io_info_t { void *status; hg_bulk_t *bulk_handle; void *buf_ptr; - size_t buf_size; + size_t nelmts; + size_t type_size; + struct H5S_t *space; } H5VL_iod_io_info_t; H5_DLL herr_t H5VL_iod_request_delete(H5VL_iod_file_t *file, H5VL_iod_request_t *request); diff --git a/src/H5VLiod_server.c b/src/H5VLiod_server.c index dfd3516..63c60c1 100644 --- a/src/H5VLiod_server.c +++ b/src/H5VLiod_server.c @@ -3368,11 +3368,14 @@ H5VL_iod_server_dset_read_cb(AXE_engine_t UNUSED axe_engine, if(H5Tconvert(src_id, dst_id, nelmts, buf, NULL, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed"); - cs = H5_checksum_fletcher32(buf, size); + /* calculate a checksum for the data to be sent */ + cs = H5_checksum_lookup3(buf, size, 0); + + /* MSC - check if client requested to corrupt data */ if(dxpl_id != H5P_DEFAULT && H5Pget_dxpl_inject_bad_checksum(dxpl_id, &flag) < 0) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read property list"); if(flag) { - fprintf(stderr, "Injecting a bad data value to generate a bad checksum \n"); + fprintf(stderr, "Injecting a bad data value to cause corruption \n"); buf_ptr[0] = 10; } } @@ -3391,7 +3394,7 @@ done: output.ret = ret_value; output.cs = cs; - fprintf(stderr, "Done with dset read, sending response to client\n"); + fprintf(stderr, "Done with dset read, checksum %u, sending response to client\n", cs); if(HG_SUCCESS != HG_Handler_start_output(op_data->hg_handle, &output)) HDONE_ERROR(H5E_SYM, H5E_WRITEERROR, FAIL, "can't send result of write to client"); @@ -3444,6 +3447,7 @@ H5VL_iod_server_dset_write_cb(AXE_engine_t UNUSED axe_engine, size_t size, buf_size, src_size, dst_size; void *buf; size_t nelmts; + hbool_t flag = FALSE; na_addr_t source = HG_Handler_get_addr(op_data->hg_handle); herr_t ret_value = SUCCEED; @@ -3495,11 +3499,18 @@ H5VL_iod_server_dset_write_cb(AXE_engine_t UNUSED axe_engine, if(HG_SUCCESS != HG_Bulk_block_handle_free(bulk_block_handle)) HGOTO_ERROR(H5E_SYM, H5E_WRITEERROR, FAIL, "can't free bds block handle"); + /* MSC - check if client requested to corrupt data */ + if(dxpl_id != H5P_DEFAULT && H5Pget_dxpl_inject_bad_checksum(dxpl_id, &flag) < 0) + HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read property list"); + if(flag) { + ((int *)buf)[0] = 10; + } + /* If client specified a checksum, verify it */ if(dxpl_id != H5P_DEFAULT && H5Pget_dxpl_checksum(dxpl_id, &data_cs) < 0) HGOTO_ERROR(H5E_SYM, H5E_READERROR, FAIL, "can't read property list"); if(data_cs != 0) { - cs = H5_checksum_fletcher32(buf, size); + cs = H5_checksum_lookup3(buf, size, 0); if(cs != data_cs) { fprintf(stderr, "Errrr.. Network transfer Data corruption. expecting %u, got %u\n", data_cs, cs); diff --git a/src/H5checksum.c b/src/H5checksum.c index ec8b2b0..de1e491 100644 --- a/src/H5checksum.c +++ b/src/H5checksum.c @@ -430,6 +430,175 @@ done: FUNC_LEAVE_NOAPI(c) } /* end H5_checksum_lookup3() */ +uint32_t +H5_checksum_lookup4(const void *key, size_t length, H5_checksum_seed_t *cs) +{ + const uint8_t *k = (const uint8_t *)key; + uint32_t position = 0; + uint32_t a, b, c; /* internal state */ + int pos = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(key); + HDassert(length > 0); + + /* Set up the internal state */ + if(cs) { + if(!cs->a && !cs->b && !cs->c) { + a = b = c = 0xdeadbeef + ((uint32_t)cs->total_length); + /* Compute new State */ + cs->state = 12 - (length % 12); + } + else { + a = cs->a; + b = cs->b; + c = cs->c; + + /* determine position to start up on */ + if(cs->state) { + position = 12 - cs->state + 1; + } + + /* Compute new State */ + cs->state = cs->state - (length % 12); + if(cs->state < 0) { + H5_lookup3_mix(a, b, c); + cs->state = 12 + cs->state; + } + + /* Move towards mod 12 */ + switch(position) /* all the case statements fall through */ + { + case 0 : + break; + case 1 : + a+=k[pos++]; + length --; + if(0 == length) + break; + case 2 : + a+=((uint32_t)k[pos++])<<8; + length --; + if(0 == length) + break; + case 3 : + a+=((uint32_t)k[pos++])<<16; + length --; + if(0 == length) + break; + case 4 : + a+=((uint32_t)k[pos++])<<24; + length --; + if(0 == length) + break; + case 5 : + b+=k[pos++]; + length --; + if(0 == length) + break; + case 6 : + b+=((uint32_t)k[pos++])<<8; + length --; + if(0 == length) + break; + case 7 : + b+=((uint32_t)k[pos++])<<16; + length --; + if(0 == length) + break; + case 8 : + b+=((uint32_t)k[pos++])<<24; + length --; + if(0 == length) + break; + case 9 : + c+=k[pos++]; + length --; + if(0 == length) + break; + case 10: + c+=((uint32_t)k[pos++])<<8; + length --; + if(0 == length) + break; + case 11: + c+=((uint32_t)k[pos++])<<16; + length --; + if(0 == length) + break; + case 12: + c+=((uint32_t)k[pos++])<<24; + length --; + H5_lookup3_mix(a, b, c); + break; + } + } + /* capture internal state if we are done at this stage */ + if(0 == length) { + cs->a = a; + cs->b = b; + cs->c = c; + H5_lookup3_final(a, b, c); + goto done; + } + } + else + a = b = c = 0xdeadbeef + ((uint32_t)length); + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) + { + a += k[0+pos]; + a += ((uint32_t)k[1+pos])<<8; + a += ((uint32_t)k[2+pos])<<16; + a += ((uint32_t)k[3+pos])<<24; + b += k[4+pos]; + b += ((uint32_t)k[5+pos])<<8; + b += ((uint32_t)k[6+pos])<<16; + b += ((uint32_t)k[7+pos])<<24; + c += k[8+pos]; + c += ((uint32_t)k[9+pos])<<8; + c += ((uint32_t)k[10+pos])<<16; + c += ((uint32_t)k[11+pos])<<24; + H5_lookup3_mix(a, b, c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch(length) /* all the case statements fall through */ + { + case 12: c+=((uint32_t)k[11+pos])<<24; + case 11: c+=((uint32_t)k[10+pos])<<16; + case 10: c+=((uint32_t)k[9+pos])<<8; + case 9 : c+=k[8+pos]; + case 8 : b+=((uint32_t)k[7+pos])<<24; + case 7 : b+=((uint32_t)k[6+pos])<<16; + case 6 : b+=((uint32_t)k[5+pos])<<8; + case 5 : b+=k[4+pos]; + case 4 : a+=((uint32_t)k[3+pos])<<24; + case 3 : a+=((uint32_t)k[2+pos])<<16; + case 2 : a+=((uint32_t)k[1+pos])<<8; + case 1 : a+=k[pos]; + break; + case 0 : goto done; + } + + /* capture internal state */ + if(cs) { + cs->a = a; + cs->b = b; + cs->c = c; + } + + H5_lookup3_final(a, b, c); + +done: + FUNC_LEAVE_NOAPI(c) +} /* end H5_checksum_lookup3() */ + /*------------------------------------------------------------------------- * Function: H5_checksum_metadata diff --git a/src/H5private.h b/src/H5private.h index 46c02e7..42ea519 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -2361,6 +2361,7 @@ H5_DLL int H5Z_term_interface(void); H5_DLL uint32_t H5_checksum_fletcher32(const void *data, size_t len); H5_DLL uint32_t H5_checksum_crc(const void *data, size_t len); H5_DLL uint32_t H5_checksum_lookup3(const void *data, size_t len, uint32_t initval); +H5_DLL uint32_t H5_checksum_lookup4(const void *data, size_t len, H5_checksum_seed_t *cs); H5_DLL uint32_t H5_checksum_metadata(const void *data, size_t len, uint32_t initval); H5_DLL uint32_t H5_hash_string(const char *str); diff --git a/src/H5public.h b/src/H5public.h index deb4587..d0f3e78 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -301,6 +301,15 @@ typedef struct H5_ih_info_t { hsize_t heap_size; } H5_ih_info_t; +/* Internal Checksum state */ +typedef struct H5_checksum_seed_t { + uint32_t a; + uint32_t b; + uint32_t c; + int32_t state; + size_t total_length; +} H5_checksum_seed_t; + /* Functions in H5.c */ H5_DLL herr_t H5open(void); H5_DLL herr_t H5close(void); -- cgit v0.12