From b9f277756bd66cd60dd2150e9800598fe3757135 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 24 Feb 2014 15:40:45 -0500 Subject: [svn-r24734] - Add iteration feature at the server with a callback on every object under the starting location. - Implement view creation for element region on all dataset under a group. - Add the creation propertly list to the token buffer since they are used now in the FF plugin. --- examples/h5ff_client_view.c | 185 ++++++++++++++++++++++++++++++++++++++------ src/H5VLiod.c | 130 +++++++++++++++++++++++++++---- src/H5VLiod_server.h | 8 ++ src/H5VLiod_util.c | 180 ++++++++++++++++++++++++++++++++++++++++++ src/H5VLiod_view.c | 138 ++++++++++++++++++++++++++++++--- 5 files changed, 594 insertions(+), 47 deletions(-) diff --git a/examples/h5ff_client_view.c b/examples/h5ff_client_view.c index 8056c08..5b7cc78 100644 --- a/examples/h5ff_client_view.c +++ b/examples/h5ff_client_view.c @@ -18,7 +18,8 @@ write_dataset(const char *file_name, const char *dataset_name, hsize_t total, hsize_t ncomponents, hid_t datatype_id, hsize_t ntuples, hsize_t start, void *buf) { - hid_t file_id, dataset_id, view_id; + hid_t file_id, view_id; + hid_t did1, did2, did3, gid1; hid_t file_space_id, mem_space_id; hid_t tid1, rid1, rid2, trspl_id; hid_t fapl_id; @@ -28,13 +29,13 @@ write_dataset(const char *file_name, const char *dataset_name, int rank = (ncomponents == 1) ? 1 : 2; uint64_t version; herr_t ret; - void *dset_token1; - size_t token_size1; + void *dset_token1, *dset_token2, *dset_token3; + size_t token_size1, token_size2, token_size3; double lower_bound1 = 39.1, upper_bound1 = 42.1; int lower_bound2 = 295, upper_bound2 = 298; hid_t query_id1, query_id2, query_id3, query_id4, query_id5, query_id6; hid_t query_id; - MPI_Request mpi_reqs[2]; + MPI_Request mpi_reqs[6]; /* Choose the IOD VOL plugin to use with this file. */ fapl_id = H5Pcreate(H5P_FILE_ACCESS); @@ -69,46 +70,97 @@ write_dataset(const char *file_name, const char *dataset_name, assert(file_space_id); if(0 == my_rank) { + /* create a group */ + gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, + tid1, H5_EVENT_STACK_NULL); + assert(gid1 > 0); + + /* Create a dataset. */ + did1 = H5Dcreate_ff(gid1, "D1", datatype_id, file_space_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL); + assert(did1); + /* Create a dataset. */ - dataset_id = H5Dcreate_ff(file_id, dataset_name, datatype_id, file_space_id, + did2 = H5Dcreate_ff(gid1, "D2", datatype_id, file_space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL); - assert(dataset_id); + assert(did2); + + /* Create a dataset. */ + did3 = H5Dcreate_ff(gid1, "D3", datatype_id, file_space_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL); + assert(did3); + + ret = H5Gclose_ff(gid1, H5_EVENT_STACK_NULL); + assert(0 == ret); /* get the token size of each dset */ - ret = H5Oget_token(dataset_id, NULL, &token_size1); + ret = H5Oget_token(did1, NULL, &token_size1); + assert(0 == ret); + ret = H5Oget_token(did2, NULL, &token_size2); + assert(0 == ret); + ret = H5Oget_token(did3, NULL, &token_size3); assert(0 == ret); /* allocate buffers for each token */ dset_token1 = malloc(token_size1); + dset_token2 = malloc(token_size2); + dset_token3 = malloc(token_size3); /* get the token buffer */ - ret = H5Oget_token(dataset_id, dset_token1, &token_size1); + ret = H5Oget_token(did1, dset_token1, &token_size1); + assert(0 == ret); + ret = H5Oget_token(did2, dset_token2, &token_size2); + assert(0 == ret); + ret = H5Oget_token(did3, dset_token3, &token_size3); assert(0 == ret); /* bcast the token sizes and the tokens */ MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); - MPI_Ibcast(dset_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, + MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]); - MPI_Waitall(2, mpi_reqs, MPI_STATUS_IGNORE); + MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[2]); + + MPI_Ibcast(dset_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[3]); + MPI_Ibcast(dset_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[4]); + MPI_Ibcast(dset_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[5]); + MPI_Waitall(6, mpi_reqs, MPI_STATUS_IGNORE); } else { /* recieve the token size */ MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); - MPI_Waitall(1, mpi_reqs, MPI_STATUS_IGNORE); + MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[1]); + MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[2]); + MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE); /* allocate buffer for token */ dset_token1 = malloc(token_size1); + dset_token2 = malloc(token_size2); + dset_token3 = malloc(token_size3); /* recieve the token */ MPI_Ibcast(dset_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); - MPI_Waitall(1, mpi_reqs, MPI_STATUS_IGNORE); + MPI_Ibcast(dset_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[1]); + MPI_Ibcast(dset_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, + &mpi_reqs[2]); + MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE); - dataset_id = H5Oopen_by_token(dset_token1, tid1, H5_EVENT_STACK_NULL); + did1 = H5Oopen_by_token(dset_token1, tid1, H5_EVENT_STACK_NULL); + did2 = H5Oopen_by_token(dset_token2, tid1, H5_EVENT_STACK_NULL); + did3 = H5Oopen_by_token(dset_token3, tid1, H5_EVENT_STACK_NULL); } free(dset_token1); + free(dset_token2); + free(dset_token3); mem_space_id = H5Screate_simple(rank, count, NULL); assert(mem_space_id); @@ -118,8 +170,16 @@ write_dataset(const char *file_name, const char *dataset_name, NULL, count, NULL); assert(0 == ret); - /* Write the first dataset. */ - ret = H5Dwrite_ff(dataset_id, datatype_id, mem_space_id, file_space_id, + /* Write to the datasets. */ + ret = H5Dwrite_ff(did1, datatype_id, mem_space_id, file_space_id, + H5P_DEFAULT, buf, tid1, H5_EVENT_STACK_NULL); + assert(0 == ret); + + ret = H5Dwrite_ff(did2, datatype_id, mem_space_id, file_space_id, + H5P_DEFAULT, buf, tid1, H5_EVENT_STACK_NULL); + assert(0 == ret); + + ret = H5Dwrite_ff(did3, datatype_id, mem_space_id, file_space_id, H5P_DEFAULT, buf, tid1, H5_EVENT_STACK_NULL); assert(0 == ret); @@ -128,9 +188,16 @@ write_dataset(const char *file_name, const char *dataset_name, assert(0 == ret); /* Finish transaction 1. */ - ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_STACK_NULL); + ret = H5TRfinish(tid1, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL); assert(0 == ret); + MPI_Barrier(MPI_COMM_WORLD); + /* acquire container version 1 - EXACT. */ + version = 1; + rid2 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL); + assert(rid2 > 0); + assert(1 == version); + /* release container version 0. */ ret = H5RCrelease(rid1, H5_EVENT_STACK_NULL); assert(0 == ret); @@ -158,7 +225,8 @@ write_dataset(const char *file_name, const char *dataset_name, query_id = H5Qcombine(query_id3, H5Q_COMBINE_OR, query_id6); assert(query_id); - view_id = H5Vcreate_ff(dataset_id, query_id, H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL); + /* create a view on D1 */ + view_id = H5Vcreate_ff(did1, query_id, H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL); assert(view_id > 0); { @@ -169,13 +237,13 @@ write_dataset(const char *file_name, const char *dataset_name, hsize_t r_dims[2]; H5Qclose(query_id); - ret = H5Dclose_ff(dataset_id, H5_EVENT_STACK_NULL); + ret = H5Dclose_ff(did1, H5_EVENT_STACK_NULL); assert(0 == ret); - ret = H5Vget_location_ff(view_id, &dataset_id, H5_EVENT_STACK_NULL); + ret = H5Vget_location_ff(view_id, &did1, H5_EVENT_STACK_NULL); assert(0 == ret); - ret = H5Dclose_ff(dataset_id, H5_EVENT_STACK_NULL); + ret = H5Dclose_ff(did1, H5_EVENT_STACK_NULL); assert(0 == ret); ret = H5Vget_query(view_id, &query_id); @@ -187,7 +255,7 @@ write_dataset(const char *file_name, const char *dataset_name, assert(0 == obj_count); assert(1 == reg_count); - ret = H5Vget_elem_regions_ff(view_id, 0, 1, &dataset_id, + ret = H5Vget_elem_regions_ff(view_id, 0, 1, &did1, ®ion_space, H5_EVENT_STACK_NULL); assert(0 == ret); @@ -198,7 +266,10 @@ write_dataset(const char *file_name, const char *dataset_name, assert(ncomponents == r_dims[1]); num_points = H5Sget_select_elem_npoints(region_space); - assert(9 == num_points); + if(my_size > 1) + assert(15 == num_points); + else + assert(9 == num_points); ret = H5Sclose(region_space); assert(0 == ret); @@ -206,6 +277,66 @@ write_dataset(const char *file_name, const char *dataset_name, H5Vclose(view_id); + gid1 = H5Gopen_ff(file_id, "G1", H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL); + assert(gid1 > 0); + + /* create a view on all datasets under G1 */ + view_id = H5Vcreate_ff(gid1, query_id, H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL); + assert(view_id > 0); + + { + hsize_t attr_count, obj_count, reg_count, i; + hssize_t num_points; + hid_t regions[3]; + hid_t did[3]; + int r_ndims; + hsize_t r_dims[2]; + + H5Qclose(query_id); + ret = H5Gclose_ff(gid1, H5_EVENT_STACK_NULL); + assert(0 == ret); + + ret = H5Vget_location_ff(view_id, &gid1, H5_EVENT_STACK_NULL); + assert(0 == ret); + assert(gid1 > 0); + + ret = H5Vget_query(view_id, &query_id); + assert(0 == ret); + + ret = H5Vget_counts(view_id, &attr_count, &obj_count, ®_count); + assert(0 == ret); + assert(0 == attr_count); + assert(0 == obj_count); + assert(3 == reg_count); + + ret = H5Vget_elem_regions_ff(view_id, 0, reg_count, did, regions, H5_EVENT_STACK_NULL); + assert(0 == ret); + + for(i=0 ; i 0); + ret = H5Dclose_ff(did[i], H5_EVENT_STACK_NULL); + assert(0 == ret); + + r_ndims = H5Sget_simple_extent_dims(regions[i], r_dims, NULL); + + assert(2 == r_ndims); + assert(total == r_dims[0]); + assert(ncomponents == r_dims[1]); + + num_points = H5Sget_select_elem_npoints(regions[i]); + if(my_size > 1) + assert(15 == num_points); + else + assert(9 == num_points); + + ret = H5Sclose(regions[i]); + assert(0 == ret); + } + } + + H5Vclose(view_id); + H5Qclose(query_id); H5Qclose(query_id6); H5Qclose(query_id5); @@ -214,8 +345,14 @@ write_dataset(const char *file_name, const char *dataset_name, H5Qclose(query_id2); H5Qclose(query_id1); - /* Close the first dataset. */ - ret = H5Dclose_ff(dataset_id, H5_EVENT_STACK_NULL); + ret = H5Gclose_ff(gid1, H5_EVENT_STACK_NULL); + assert(0 == ret); + + ret = H5Dclose_ff(did1, H5_EVENT_STACK_NULL); + assert(0 == ret); + ret = H5Dclose_ff(did2, H5_EVENT_STACK_NULL); + assert(0 == ret); + ret = H5Dclose_ff(did3, H5_EVENT_STACK_NULL); assert(0 == ret); ret = H5Sclose(file_space_id); assert(0 == ret); diff --git a/src/H5VLiod.c b/src/H5VLiod.c index 1dbd8d0..bd5be88 100644 --- a/src/H5VLiod.c +++ b/src/H5VLiod.c @@ -21,6 +21,8 @@ * by the function shipper. */ +#define H5P_PACKAGE /*suppress error about including H5Ppkg */ + /* Interface initialization */ #define H5_INTERFACE_INIT_FUNC H5VL_iod_init_interface @@ -31,6 +33,7 @@ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5Ppkg.h" /* Property lists */ #include "H5Sprivate.h" /* Dataspaces */ #include "H5VLprivate.h" /* VOL plugins */ #include "H5VLiod.h" /* Iod VOL plugin */ @@ -6097,7 +6100,18 @@ H5VL_iod_obj_open_token(const void *token, H5TR_t *tr, H5I_type_t *opened_type, dset->remote_dset.attrkv_id = attrkv_id; dset->dapl_id = H5Pcopy(H5P_DATASET_ACCESS_DEFAULT); - dset->remote_dset.dcpl_id = H5Pcopy(H5P_DATASET_CREATE_DEFAULT); + + /* decode creation property list */ + { + size_t plist_size; + + HDmemcpy(&plist_size, buf_ptr, sizeof(size_t)); + buf_ptr += sizeof(size_t); + /* Create plist by decoding buffer */ + if((dset->remote_dset.dcpl_id = H5P__decode((const void *)buf_ptr)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTDECODE, NULL, "can't decode object"); + buf_ptr += plist_size; + } /* decode dtype */ { @@ -6166,7 +6180,18 @@ H5VL_iod_obj_open_token(const void *token, H5TR_t *tr, H5I_type_t *opened_type, dtype->remote_dtype.attrkv_id = attrkv_id; dtype->tapl_id = H5Pcopy(H5P_DATATYPE_ACCESS_DEFAULT); - dtype->remote_dtype.tcpl_id = H5Pcopy(H5P_DATATYPE_CREATE_DEFAULT); + + /* decode creation property list */ + { + size_t plist_size; + + HDmemcpy(&plist_size, buf_ptr, sizeof(size_t)); + buf_ptr += sizeof(size_t); + /* Create plist by decoding buffer */ + if((dtype->remote_dtype.tcpl_id = H5P__decode((const void *)buf_ptr)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTDECODE, NULL, "can't decode object"); + buf_ptr += plist_size; + } /* decode dtype */ { @@ -6220,9 +6245,20 @@ H5VL_iod_obj_open_token(const void *token, H5TR_t *tr, H5I_type_t *opened_type, grp->remote_group.mdkv_id = mdkv_id; grp->remote_group.attrkv_id = attrkv_id; - grp->remote_group.gcpl_id = H5Pcopy(H5P_GROUP_CREATE_DEFAULT); grp->gapl_id = H5Pcopy(H5P_GROUP_ACCESS_DEFAULT); + /* decode creation property list */ + { + size_t plist_size; + + HDmemcpy(&plist_size, buf_ptr, sizeof(size_t)); + buf_ptr += sizeof(size_t); + /* Create plist by decoding buffer */ + if((grp->remote_group.gcpl_id = H5P__decode((const void *)buf_ptr)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTDECODE, NULL, "can't decode object"); + buf_ptr += plist_size; + } + /* set the input structure for the HG encode routine */ input.coh = tr->file->remote_file.coh; input.iod_id = iod_id; @@ -6259,9 +6295,20 @@ H5VL_iod_obj_open_token(const void *token, H5TR_t *tr, H5I_type_t *opened_type, map->remote_map.mdkv_id = mdkv_id; map->remote_map.attrkv_id = attrkv_id; - map->remote_map.mcpl_id = H5Pcopy(H5P_MAP_CREATE_DEFAULT); map->mapl_id = H5Pcopy(H5P_MAP_ACCESS_DEFAULT); + /* decode creation property list */ + { + size_t plist_size; + + HDmemcpy(&plist_size, buf_ptr, sizeof(size_t)); + buf_ptr += sizeof(size_t); + /* Create plist by decoding buffer */ + if((map->remote_map.mcpl_id = H5P__decode((const void *)buf_ptr)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTDECODE, NULL, "can't decode object"); + buf_ptr += plist_size; + } + /* decode key_type */ { H5T_t *dt = NULL; @@ -6430,9 +6477,11 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) iod_obj_id_t iod_id, mdkv_id, attrkv_id; H5O_type_t type; uint8_t *buf_ptr = (uint8_t *)token; - size_t dt_size = 0, space_size = 0; + size_t dt_size = 0, space_size = 0, plist_size = 0; H5T_t *dt = NULL; H5S_t *space = NULL; + H5P_genplist_t *plist = NULL; + hid_t cpl_id; size_t keytype_size = 0, valtype_size; H5T_t *kt = NULL, *vt = NULL; herr_t ret_value = SUCCEED; /* Return value */ @@ -6443,11 +6492,23 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) switch(obj->obj_type) { case H5I_GROUP: - iod_id = ((const H5VL_iod_group_t *)obj)->remote_group.iod_id; - mdkv_id = ((const H5VL_iod_group_t *)obj)->remote_group.mdkv_id; - attrkv_id = ((const H5VL_iod_group_t *)obj)->remote_group.attrkv_id; - type = H5O_TYPE_GROUP; - break; + { + H5VL_iod_group_t *grp = (H5VL_iod_group_t *)obj; + + iod_id = grp->remote_group.iod_id; + mdkv_id = grp->remote_group.mdkv_id; + attrkv_id = grp->remote_group.attrkv_id; + type = H5O_TYPE_GROUP; + + cpl_id = grp->remote_group.gcpl_id; + if(NULL == (plist = (H5P_genplist_t *)H5I_object_verify(cpl_id, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + if(H5P__encode(plist, TRUE, NULL, &plist_size) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, FAIL, "can't encode plist") + + *token_size += plist_size + sizeof(size_t); + break; + } case H5I_DATASET: { H5VL_iod_dset_t *dset = (H5VL_iod_dset_t *)obj; @@ -6470,7 +6531,13 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) if(H5S_encode(space, NULL, &space_size) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTENCODE, FAIL, "can't encode dataspace") - *token_size += dt_size + space_size + sizeof(size_t)*2; + cpl_id = dset->remote_dset.dcpl_id; + if(NULL == (plist = (H5P_genplist_t *)H5I_object_verify(cpl_id, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list"); + if(H5P__encode(plist, TRUE, NULL, &plist_size) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, FAIL, "can't encode plist") + + *token_size += plist_size + dt_size + space_size + sizeof(size_t)*3; break; } @@ -6490,7 +6557,13 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) if(H5T_encode(dt, NULL, &dt_size) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, FAIL, "can't encode datatype") - *token_size += dt_size + sizeof(size_t); + cpl_id = dtype->remote_dtype.tcpl_id; + if(NULL == (plist = (H5P_genplist_t *)H5I_object_verify(cpl_id, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list"); + if(H5P__encode(plist, TRUE, NULL, &plist_size) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, FAIL, "can't encode plist") + + *token_size += dt_size + plist_size + sizeof(size_t)*2; break; } @@ -6517,7 +6590,13 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) if(H5T_encode(vt, NULL, &valtype_size) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, FAIL, "can't encode datatype") - *token_size += keytype_size + valtype_size + sizeof(size_t)*2; + cpl_id = map->remote_map.mcpl_id; + if(NULL == (plist = (H5P_genplist_t *)H5I_object_verify(cpl_id, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list"); + if(H5P__encode(plist, TRUE, NULL, &plist_size) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, FAIL, "can't encode plist") + + *token_size += keytype_size + valtype_size + plist_size + sizeof(size_t)*3; break; } @@ -6556,8 +6635,19 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) switch(obj->obj_type) { case H5I_GROUP: + HDmemcpy(buf_ptr, &plist_size, sizeof(size_t)); + buf_ptr += sizeof(size_t); + if(H5P__encode(plist, TRUE, buf_ptr, &plist_size) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTENCODE, FAIL, "can't encode dataspace") + buf_ptr += plist_size; break; case H5I_DATASET: + HDmemcpy(buf_ptr, &plist_size, sizeof(size_t)); + buf_ptr += sizeof(size_t); + if(H5P__encode(plist, TRUE, buf_ptr, &plist_size) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTENCODE, FAIL, "can't encode dataspace") + buf_ptr += plist_size; + HDmemcpy(buf_ptr, &dt_size, sizeof(size_t)); buf_ptr += sizeof(size_t); if(H5T_encode(dt, buf_ptr, &dt_size) < 0) @@ -6571,6 +6661,12 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) buf_ptr += space_size; break; case H5I_DATATYPE: + HDmemcpy(buf_ptr, &plist_size, sizeof(size_t)); + buf_ptr += sizeof(size_t); + if(H5P__encode(plist, TRUE, buf_ptr, &plist_size) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTENCODE, FAIL, "can't encode dataspace") + buf_ptr += plist_size; + HDmemcpy(buf_ptr, &dt_size, sizeof(size_t)); buf_ptr += sizeof(size_t); if(H5T_encode(dt, buf_ptr, &dt_size) < 0) @@ -6578,6 +6674,12 @@ H5VL_iod_get_token(void *_obj, void *token, size_t *token_size) buf_ptr += dt_size; break; case H5I_MAP: + HDmemcpy(buf_ptr, &plist_size, sizeof(size_t)); + buf_ptr += sizeof(size_t); + if(H5P__encode(plist, TRUE, buf_ptr, &plist_size) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTENCODE, FAIL, "can't encode dataspace") + buf_ptr += plist_size; + HDmemcpy(buf_ptr, &keytype_size, sizeof(size_t)); buf_ptr += sizeof(size_t); if(H5T_encode(kt, buf_ptr, &keytype_size) < 0) @@ -9907,7 +10009,7 @@ H5VL_iod_view_create(void *_obj, hid_t query_id, hid_t vcpl_id, hid_t rcxt_id, v /* store the query ID */ view->query_id = query_id; - if(H5I_inc_ref(query_id, FALSE) < 0) + if(H5I_inc_ref(query_id, TRUE) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTINC, NULL, "can't increment ID ref count"); /* copy property list */ diff --git a/src/H5VLiod_server.h b/src/H5VLiod_server.h index d49e323..7ebb609 100644 --- a/src/H5VLiod_server.h +++ b/src/H5VLiod_server.h @@ -90,6 +90,10 @@ extern hg_id_t H5VL_EFF_CLOSE_CONTAINER; extern hg_id_t H5VL_EFF_ANALYSIS_FARM; extern hg_id_t H5VL_EFF_ANALYSIS_FARM_TRANSFER; +/* Define the operator function pointer for H5Diterate() */ +typedef herr_t (*H5VL_operator_t)(iod_handle_t coh, iod_obj_id_t obj_id, iod_trans_id_t rtid, + H5I_type_t obj_type, uint32_t cs_scope, void *operator_data); + H5_DLL void EFF__mercury_register_callbacks(void); H5_DLL int H5VL_iod_server_analysis_execute(hg_handle_t handle); @@ -444,6 +448,10 @@ H5_DLL herr_t H5VL__iod_server_final_io(iod_handle_t iod_oh, hid_t space_id, siz hbool_t write_op, void *buf, size_t buf_size, iod_checksum_t cs, uint32_t cs_scope, iod_trans_id_t tid); +H5_DLL herr_t H5VL_iod_server_iterate(iod_handle_t coh, iod_obj_id_t obj_id, iod_trans_id_t rtid, + H5I_type_t obj_type, uint32_t cs_scope, + H5VL_operator_t op, void *op_data); + H5_DLL herr_t H5VL__iod_get_query_data_cb(void *elem, hid_t type_id, unsigned ndim, const hsize_t *point, void *_udata); #endif /* H5_HAVE_EFF */ diff --git a/src/H5VLiod_util.c b/src/H5VLiod_util.c index c577ecb..a67d7b8 100644 --- a/src/H5VLiod_util.c +++ b/src/H5VLiod_util.c @@ -22,6 +22,9 @@ #ifdef H5_HAVE_EFF +static H5I_type_t H5VL__iod_get_h5_obj_type(iod_obj_id_t oid, iod_handle_t coh, + iod_trans_id_t rtid, uint32_t cs_scope); + /* * Programmer: Mohamad Chaarawi * February, 2013 @@ -1243,6 +1246,183 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5VL_iod_verify_kv_pair */ + +/*------------------------------------------------------------------------- + * Function: H5VL__iod_get_h5_obj_type + * + * Purpose: Function to retrieve the HDF5 object type of an IOD object. + * + * Return: Success: SUCCEED + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static H5I_type_t +H5VL__iod_get_h5_obj_type(iod_obj_id_t oid, iod_handle_t coh, iod_trans_id_t rtid, uint32_t cs_scope) +{ + iod_handle_t mdkv_oh, oh; + H5I_type_t obj_type; + iod_obj_type_t iod_type; + herr_t ret_value = -1; + + FUNC_ENTER_NOAPI_NOINIT + + iod_type = IOD_OBJID_GETTYPE(oid); + + if(IOD_OBJ_ARRAY == iod_type) + obj_type = H5I_DATASET; + else if(IOD_OBJ_BLOB == iod_type) + obj_type = H5I_DATATYPE; + else { + scratch_pad sp; + iod_checksum_t sp_cs = 0; + + if (iod_obj_open_read(coh, oid, rtid, NULL /*hints*/, &oh, NULL) < 0) + HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open object"); + + /* get scratch pad of the object */ + if(iod_obj_get_scratch(oh, rtid, &sp, &sp_cs, NULL) < 0) + HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't get scratch pad for object"); + if(sp_cs && (cs_scope & H5_CHECKSUM_IOD)) { + /* verify scratch pad integrity */ + if(H5VL_iod_verify_scratch_pad(&sp, sp_cs) < 0) + HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "Scratch Pad failed integrity check"); + } + + /* open the metadata KV */ + if (iod_obj_open_read(coh, sp[0], rtid, NULL /*hints*/, &mdkv_oh, NULL) < 0) + HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't open MDKV"); + + if(H5VL_iod_get_metadata(mdkv_oh, rtid, H5VL_IOD_OBJECT_TYPE, H5VL_IOD_KEY_OBJ_TYPE, + cs_scope, NULL, &obj_type) < 0) + HGOTO_ERROR2(H5E_SYM, H5E_CANTGET, FAIL, "failed to retrieve link count"); + + if(iod_obj_close(mdkv_oh, NULL, NULL) < 0) + HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close current object handle"); + if(iod_obj_close(oh, NULL, NULL) < 0) + HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't close current object handle"); + } + + ret_value = obj_type; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5VL__iod_get_h5_obj_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5VL_iod_server_iterate + * + * Purpose: + * + * Return: Success: SUCCEED + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5VL_iod_server_iterate(iod_handle_t coh, iod_obj_id_t obj_id, iod_trans_id_t rtid, + H5I_type_t obj_type, uint32_t cs_scope, + H5VL_operator_t op, void *op_data) +{ + iod_handle_t obj_oh; + herr_t ret; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + ret = (*op)(coh, obj_id, rtid, obj_type, cs_scope, op_data); + + /* Get the object type, if it is not a group do not check for links */ + if(H5I_GROUP == obj_type) { + int num_entries; + + /* Get the object ID and iterate into every member in the group */ + if (iod_obj_open_read(coh, obj_id, rtid, NULL, &obj_oh, NULL) < 0) + HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't open current group"); + + ret = iod_kv_get_num(obj_oh, rtid, &num_entries, NULL); + if(ret != 0) + HGOTO_ERROR_IOD(ret, FAIL, "can't get number of KV entries"); + + if(0 != num_entries) { + iod_kv_params_t *kvs = NULL; + iod_kv_t *kv = NULL; + iod_checksum_t *oid_cs = NULL; + iod_ret_t *oid_ret = NULL; + int i; + + kvs = (iod_kv_params_t *)malloc(sizeof(iod_kv_params_t) * (size_t)num_entries); + kv = (iod_kv_t *)malloc(sizeof(iod_kv_t) * (size_t)num_entries); + oid_cs = (iod_checksum_t *)malloc(sizeof(iod_checksum_t) * (size_t)num_entries); + oid_ret = (iod_ret_t *)malloc(sizeof(iod_ret_t) * (size_t)num_entries); + + for(i=0 ; iregion_info.count; + + op_data->region_info.tokens = (binary_buf_t *)realloc(op_data->region_info.tokens, + (i+1) * sizeof(binary_buf_t)); + op_data->region_info.regions = (hid_t *)realloc(op_data->region_info.regions, + (i+1) * sizeof(hid_t)); + + if((op_data->region_info.regions[i] = + H5VL__iod_get_elmt_region(coh, obj_id, rtid, op_data->query_id, op_data->vcpl_id, + cs_scope, &op_data->region_info.tokens[i])) < 0) + HGOTO_ERROR2(H5E_FILE, H5E_CANTINIT, FAIL, "can't get region from query"); + + op_data->region_info.count ++; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + /*------------------------------------------------------------------------- * Function: H5VL_iod_server_view_create_cb * @@ -65,10 +102,7 @@ H5VL_iod_server_view_create_cb(AXE_engine_t UNUSED axe_engine, hid_t vcpl_id; iod_trans_id_t rtid = input->rcxt_num; uint32_t cs_scope = input->cs_scope; - iod_handle_t mdkv_oh; - scratch_pad sp; - iod_ret_t ret; - hid_t region = FAIL; + hsize_t i; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -96,7 +130,33 @@ H5VL_iod_server_view_create_cb(AXE_engine_t UNUSED axe_engine, output.obj_info.tokens = NULL; output.attr_info.count = 0; output.attr_info.tokens = NULL; - + } + else if (H5I_GROUP == obj_type) { + H5VL_view_op_t udata; + + udata.query_id = query_id; + udata.vcpl_id = vcpl_id; + + udata.region_info.count = 0; + udata.region_info.tokens = NULL; + udata.region_info.regions = NULL; + udata.obj_info.count = 0; + udata.obj_info.tokens = NULL; + udata.attr_info.count = 0; + udata.attr_info.tokens = NULL; + + if(H5VL_iod_server_iterate(coh, loc_id, rtid, obj_type, cs_scope, + H5VL__iod_view_iterate_cb, &udata) < 0) + HGOTO_ERROR2(H5E_SYM, H5E_CANTINIT, FAIL, "can't iterate to create group"); + + output.region_info.count = udata.region_info.count; + output.region_info.tokens = udata.region_info.tokens; + output.region_info.regions = udata.region_info.regions; + output.valid_view = TRUE; + output.obj_info.count = 0; + output.obj_info.tokens = NULL; + output.attr_info.count = 0; + output.attr_info.tokens = NULL; } else { /* MSC - for now this is only what is supported */ @@ -117,6 +177,31 @@ done: HG_Handler_start_output(op_data->hg_handle, &output); + for(i=0 ; i