From ad72021fd1b22c0e83053a30b3fcd66c40e29a84 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 17 Mar 2014 12:37:27 -0500 Subject: [svn-r24813] Add H5Xpublic/H5Vpublic to hdf5.h Add index test in testff Add indexing query callback to H5Vcreate Cleanup --- src/H5FF.c | 8 +- src/H5V.c | 46 +++++++++- src/H5VLiod.c | 4 +- src/H5VLiod_client.h | 4 +- src/H5VLiod_common.h | 1 + src/hdf5.h | 4 + testff/CMakeLists.txt | 2 +- testff/h5ff_client_index.c | 206 +++++++++++++++++++++------------------------ 8 files changed, 155 insertions(+), 120 deletions(-) diff --git a/src/H5FF.c b/src/H5FF.c index 122acd2..fa1acc3 100644 --- a/src/H5FF.c +++ b/src/H5FF.c @@ -800,7 +800,7 @@ H5Dopen_ff(hid_t loc_id, const char *name, hid_t dapl_id, hid_t rcxt_id, hid_t e size_t metadata_size; size_t idx_count; void *metadata; - H5P_genplist_t *xxpl_plist; /* Property list pointer */ + H5P_genplist_t *xapl_plist; /* Property list pointer */ hid_t xapl_id = H5P_INDEX_ACCESS_DEFAULT; /* Get index info if present */ @@ -809,10 +809,10 @@ H5Dopen_ff(hid_t loc_id, const char *name, hid_t dapl_id, hid_t rcxt_id, hid_t e HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get index info for dataset"); if (idx_count) { - /* store the read context ID in the xxpl */ - if(NULL == (xxpl_plist = (H5P_genplist_t *)H5I_object(xapl_id))) + /* store the read context ID in the xapl */ + if(NULL == (xapl_plist = (H5P_genplist_t *)H5I_object(xapl_id))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); - if(H5P_set(xxpl_plist, H5VL_CONTEXT_ID, &rcxt_id) < 0) + if(H5P_set(xapl_plist, H5VL_CONTEXT_ID, &rcxt_id) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set property value for trans_id"); if (NULL == (idx_class = H5X_registered(plugin_id))) diff --git a/src/H5V.c b/src/H5V.c index 5da1eeb..00e2331 100644 --- a/src/H5V.c +++ b/src/H5V.c @@ -33,6 +33,9 @@ #include "H5Vprivate.h" /* Views */ #include "H5VLprivate.h" /* VOL plugins */ #include "H5VLiod_client.h" /* IOD VOL plugin */ +#ifdef H5_HAVE_INDEXING +#include "H5Xprivate.h" /* Indexing */ +#endif #ifdef H5_HAVE_EFF @@ -305,6 +308,10 @@ H5Vcreate_ff(hid_t loc_id, hid_t query_id, hid_t vcpl_id, hid_t rcxt_id, hid_t e void **req = NULL; /* pointer to plugin generate requests (Stays NULL if plugin does not support async */ void *view = NULL; /* pointer to view object created */ void *obj = NULL; /* object token of loc_id */ +#ifdef H5_HAVE_INDEXING + void *idx_handle = NULL; /* index */ +#endif + hid_t dataspace_id; H5VL_t *vol_plugin; /* VOL plugin information */ hid_t ret_value; @@ -339,8 +346,41 @@ H5Vcreate_ff(hid_t loc_id, hid_t query_id, hid_t vcpl_id, hid_t rcxt_id, hid_t e vol_plugin->nrefs ++; } +#ifdef H5_HAVE_INDEXING + /* Try to get indexing info (only for one dataset now) */ + if (H5I_object_verify(loc_id, H5I_DATASET) && + (NULL != (idx_handle = H5VL_iod_dataset_get_index(obj)))) { + H5X_class_t *idx_class = NULL; + H5P_genplist_t *xxpl_plist; /* Property list pointer */ + hid_t xxpl_id = H5P_INDEX_XFER_DEFAULT; + unsigned plugin_id; + + if (!(plugin_id = H5VL_iod_dataset_get_index_plugin_id(obj))) + HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get index plugin ID from dataset"); + if (NULL == (idx_class = H5X_registered(plugin_id))) + HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get index plugin class"); + + /* store the read context ID in the xxpl */ + if (NULL == (xxpl_plist = (H5P_genplist_t *) H5I_object(xxpl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); + if (H5P_set(xxpl_plist, H5VL_CONTEXT_ID, &rcxt_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set property value for trans_id"); + + if (FAIL == idx_class->query(idx_handle, query_id, xxpl_id, &dataspace_id)) + HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "cannot close index"); + } +#endif + if (!H5I_object_verify(dataspace_id, H5I_DATASPACE)) { + /* The indexing pre_selection is NULL */ + H5S_t *dataspace = H5S_create(H5S_NULL); + if((ret_value = H5I_register (H5I_DATASPACE, dataspace, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace atom"); + } + /* TODO might want to pass dataspace_id with vcpl_id */ + /* call the IOD specific private routine to create a view object */ - if(NULL == (view = H5VL_iod_view_create(obj, query_id, vcpl_id, rcxt_id, req))) + if(NULL == (view = H5VL_iod_view_create(obj, query_id, dataspace_id, + vcpl_id, rcxt_id, req))) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to create view") if(request && *req) { @@ -353,9 +393,11 @@ H5Vcreate_ff(hid_t loc_id, hid_t query_id, hid_t vcpl_id, hid_t rcxt_id, hid_t e HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize view handle") done: - if (ret_value < 0 && view) + if (ret_value < 0 && view) { if(H5V_close (view) < 0) HDONE_ERROR(H5E_SYM, H5E_CLOSEERROR, FAIL, "unable to release view") + } + H5I_dec_ref(dataspace_id); FUNC_LEAVE_API(ret_value) } /* end H5Vcreate_ff */ diff --git a/src/H5VLiod.c b/src/H5VLiod.c index 738e233..2075250 100644 --- a/src/H5VLiod.c +++ b/src/H5VLiod.c @@ -10041,7 +10041,8 @@ done: *------------------------------------------------------------------------- */ void * -H5VL_iod_view_create(void *_obj, hid_t query_id, hid_t vcpl_id, hid_t rcxt_id, void **req) +H5VL_iod_view_create(void *_obj, hid_t query_id, hid_t dataspace_id, + hid_t vcpl_id, hid_t rcxt_id, void **req) { H5VL_iod_object_t *obj = (H5VL_iod_object_t *)_obj; /* location object to create the view */ H5VL_iod_view_t *view = NULL; /* the view object that is created and passed to the user */ @@ -10098,6 +10099,7 @@ H5VL_iod_view_create(void *_obj, hid_t query_id, hid_t vcpl_id, hid_t rcxt_id, v input.loc_oh = iod_oh; input.loc_mdkv_id = mdkv_id; input.query_id = query_id; + input.dataspace_id = dataspace_id; input.vcpl_id = vcpl_id; input.obj_type = obj->obj_type; input.rcxt_num = rc->c_version; diff --git a/src/H5VLiod_client.h b/src/H5VLiod_client.h index aabf520..a5ae2f4 100644 --- a/src/H5VLiod_client.h +++ b/src/H5VLiod_client.h @@ -484,8 +484,8 @@ H5_DLL herr_t H5VL_iod_prefetch(void *obj, hid_t rcxt_id, hrpl_t *replica_id, hid_t apl_id, void **req); H5_DLL herr_t H5VL_iod_evict(void *obj, uint64_t c_version, hid_t apl_id, void **req); -H5_DLL void * H5VL_iod_view_create(void *_obj, hid_t query_id, hid_t vcpl_id, - hid_t rcxt_id, void **req); +H5_DLL void * H5VL_iod_view_create(void *_obj, hid_t query_id, hid_t dataspace_id, + hid_t vcpl_id, hid_t rcxt_id, void **req); H5_DLL herr_t H5VL_iod_view_close(H5VL_iod_view_t *view); H5_DLL herr_t H5VL_iod_analysis_execute(const char *file_name, const char *obj_name, diff --git a/src/H5VLiod_common.h b/src/H5VLiod_common.h index f0e42dd..2e7a691 100644 --- a/src/H5VLiod_common.h +++ b/src/H5VLiod_common.h @@ -756,6 +756,7 @@ MERCURY_GEN_PROC(view_create_in_t, ((iod_obj_id_t)(loc_mdkv_id)) ((uint64_t)(rcxt_num)) ((int32_t)(obj_type)) + ((hid_t)(dataspace_id)) ((hid_t)(vcpl_id)) ((hid_t)(query_id))) diff --git a/src/hdf5.h b/src/hdf5.h index c46c694..3735585 100644 --- a/src/hdf5.h +++ b/src/hdf5.h @@ -42,6 +42,10 @@ #include "H5Spublic.h" /* Dataspaces */ #include "H5Tpublic.h" /* Datatypes */ #include "H5TRpublic.h" /* Transcations */ +#include "H5Vpublic.h" /* Views */ +#ifdef H5_HAVE_INDEXING +#include "H5Xpublic.h" /* Index plugins */ +#endif #include "H5Zpublic.h" /* Data filters */ #include "H5VLpublic.h" /* VOL plugins */ diff --git a/testff/CMakeLists.txt b/testff/CMakeLists.txt index bd85255..71f395e 100644 --- a/testff/CMakeLists.txt +++ b/testff/CMakeLists.txt @@ -74,7 +74,7 @@ BUILD_H5FF_TEST(h5ff_client_analysis) BUILD_H5FF_TEST(h5ff_client_attr) BUILD_H5FF_TEST(h5ff_client_dset) BUILD_H5FF_TEST(h5ff_client_evict_deltas) -#BUILD_H5FF_TEST(h5ff_client_index) +BUILD_H5FF_TEST(h5ff_client_index) BUILD_H5FF_TEST(h5ff_client_links) BUILD_H5FF_TEST(h5ff_client_M6.2_demo) BUILD_H5FF_TEST(h5ff_client_map) diff --git a/testff/h5ff_client_index.c b/testff/h5ff_client_index.c index cd59728..d5c7264 100644 --- a/testff/h5ff_client_index.c +++ b/testff/h5ff_client_index.c @@ -14,14 +14,13 @@ static int my_rank = 0, my_size = 1; static void -write_dataset(const char *file_name, const char *dataset_name, +write_dataset(hid_t file_id, const char *dataset_name, hsize_t total, hsize_t ncomponents, hid_t datatype_id, hsize_t ntuples, hsize_t start, void *buf) { - hid_t file_id, dataset_id, view_id; + hid_t dataset_id; hid_t file_space_id, mem_space_id; - hid_t tid1, rid1, rid2, trspl_id; - hid_t fapl_id; + hid_t tid0, rid0, trspl_id; hsize_t dims[2] = {total, ncomponents}; hsize_t offset[2] = {start, 0}; hsize_t count[2] = {ntuples, ncomponents}; @@ -30,36 +29,21 @@ write_dataset(const char *file_name, const char *dataset_name, herr_t ret; void *dset_token1; size_t token_size1; - double lower_bound1 = 39.1, upper_bound1 = 42.1; - int lower_bound2 = 295, upper_bound2 = 298; - hid_t query_id1, query_id2, query_id3, query_id4, query_id5, query_id6; - hid_t query_id; MPI_Request mpi_reqs[2]; - /* Choose the IOD VOL plugin to use with this file. */ - fapl_id = H5Pcreate(H5P_FILE_ACCESS); - H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); - - /* Open an existing file. */ - file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, - H5_EVENT_STACK_NULL); - - ret = H5Pclose(fapl_id); - assert(0 == ret); - /* acquire container version 0 - EXACT. */ version = 0; - rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL); + rid0 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL); assert(0 == version); /* create transaction object */ - tid1 = H5TRcreate(file_id, rid1, (uint64_t)1); - assert(tid1); + tid0 = H5TRcreate(file_id, rid0, (uint64_t)1); + assert(tid0); trspl_id = H5Pcreate(H5P_TR_START); ret = H5Pset_trspl_num_peers(trspl_id, (unsigned int) my_size); assert(0 == ret); - ret = H5TRstart(tid1, trspl_id, H5_EVENT_STACK_NULL); + ret = H5TRstart(tid0, trspl_id, H5_EVENT_STACK_NULL); assert(0 == ret); ret = H5Pclose(trspl_id); assert(0 == ret); @@ -71,9 +55,16 @@ write_dataset(const char *file_name, const char *dataset_name, if(0 == my_rank) { /* Create a dataset. */ dataset_id = H5Dcreate_ff(file_id, dataset_name, datatype_id, file_space_id, - H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL); + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid0, H5_EVENT_STACK_NULL); assert(dataset_id); +#ifdef H5_HAVE_INDEXING + /* Add indexing information */ + ret = H5Xcreate_ff(file_id, H5X_PLUGIN_DUMMY, dataset_id, H5P_DEFAULT, + tid0, H5_EVENT_STACK_NULL); + assert(0 == ret); +#endif + /* get the token size of each dset */ ret = H5Oget_token(dataset_id, NULL, &token_size1); assert(0 == ret); @@ -88,7 +79,7 @@ write_dataset(const char *file_name, const char *dataset_name, /* bcast the token sizes and the tokens */ MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); - MPI_Ibcast(dset_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, + MPI_Ibcast(dset_token1, (int) token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]); MPI_Waitall(2, mpi_reqs, MPI_STATUS_IGNORE); } @@ -102,11 +93,11 @@ write_dataset(const char *file_name, const char *dataset_name, dset_token1 = malloc(token_size1); /* recieve the token */ - MPI_Ibcast(dset_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, + MPI_Ibcast(dset_token1, (int) token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); MPI_Waitall(1, mpi_reqs, MPI_STATUS_IGNORE); - dataset_id = H5Oopen_by_token(dset_token1, tid1, H5_EVENT_STACK_NULL); + dataset_id = H5Oopen_by_token(dset_token1, tid0, H5_EVENT_STACK_NULL); } free(dset_token1); @@ -120,122 +111,112 @@ write_dataset(const char *file_name, const char *dataset_name, /* Write the first dataset. */ ret = H5Dwrite_ff(dataset_id, datatype_id, mem_space_id, file_space_id, - H5P_DEFAULT, buf, tid1, H5_EVENT_STACK_NULL); + H5P_DEFAULT, buf, tid0, H5_EVENT_STACK_NULL); assert(0 == ret); /* Close the data space for the first dataset. */ ret = H5Sclose(mem_space_id); assert(0 == ret); - /* Finish transaction 1. */ - ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_STACK_NULL); + /* Close the first dataset. */ + ret = H5Dclose_ff(dataset_id, H5_EVENT_STACK_NULL); + assert(0 == ret); + + ret = H5Sclose(file_space_id); + assert(0 == ret); + + /* Finish transaction 0. */ + ret = H5TRfinish(tid0, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL); assert(0 == ret); /* release container version 0. */ - ret = H5RCrelease(rid1, H5_EVENT_STACK_NULL); + ret = H5RCrelease(rid0, H5_EVENT_STACK_NULL); assert(0 == ret); - MPI_Barrier(MPI_COMM_WORLD); + ret = H5RCclose(rid0); + assert(0 == ret); + + ret = H5TRclose(tid0); + assert(0 == ret); +} + +static hid_t +query_and_view(hid_t file_id, const char *dataset_name) +{ + double lower_bound1 = 39.1, upper_bound1 = 42.1; + int lower_bound2 = 295, upper_bound2 = 298; + hid_t query_id1, query_id2, query_id3, query_id4, query_id5, query_id6; + hid_t query_id, view_id; + hid_t dataset_id; + hid_t rid1; + uint64_t version; + herr_t ret; /* Create a simple query */ /* query = (39.1 < x < 42.1) || (295 < x < 298) */ query_id1 = H5Qcreate(H5Q_TYPE_DATA_ELEM, H5Q_MATCH_GREATER_THAN, H5T_NATIVE_DOUBLE, &lower_bound1); assert(query_id1); + query_id2 = H5Qcreate(H5Q_TYPE_DATA_ELEM, H5Q_MATCH_LESS_THAN, H5T_NATIVE_DOUBLE, &upper_bound1); assert(query_id2); + query_id3 = H5Qcombine(query_id1, H5Q_COMBINE_AND, query_id2); assert(query_id3); + query_id4 = H5Qcreate(H5Q_TYPE_DATA_ELEM, H5Q_MATCH_GREATER_THAN, H5T_NATIVE_INT, &lower_bound2); assert(query_id4); + query_id5 = H5Qcreate(H5Q_TYPE_DATA_ELEM, H5Q_MATCH_LESS_THAN, H5T_NATIVE_INT, &upper_bound2); assert(query_id5); + query_id6 = H5Qcombine(query_id4, H5Q_COMBINE_AND, query_id5); assert(query_id6); + query_id = H5Qcombine(query_id3, H5Q_COMBINE_OR, query_id6); assert(query_id); - view_id = H5Vcreate_ff(dataset_id, query_id, H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL); - assert(view_id > 0); - - { - hsize_t attr_count, obj_count, reg_count; - hssize_t num_points; - hid_t region_space; - int r_ndims; - hsize_t r_dims[2]; - - H5Qclose(query_id); - ret = H5Dclose_ff(dataset_id, H5_EVENT_STACK_NULL); - assert(0 == ret); - - ret = H5Vget_location_ff(view_id, &dataset_id, H5_EVENT_STACK_NULL); - assert(0 == ret); - - ret = H5Dclose_ff(dataset_id, H5_EVENT_STACK_NULL); - assert(0 == ret); - - ret = H5Vget_query(view_id, &query_id); - assert(0 == ret); - - ret = H5Vget_counts(view_id, &attr_count, &obj_count, ®_count); - assert(0 == ret); - assert(0 == attr_count); - assert(0 == obj_count); - assert(1 == reg_count); - - ret = H5Vget_elem_regions_ff(view_id, 0, 1, &dataset_id, - ®ion_space, H5_EVENT_STACK_NULL); - assert(0 == ret); + /* acquire container version 1 - EXACT. */ + version = 1; + rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL); + assert(rid1 > 0); + assert(1 == version); - r_ndims = H5Sget_simple_extent_dims(region_space, r_dims, NULL); + MPI_Barrier(MPI_COMM_WORLD); - assert(2 == r_ndims); - assert(total == r_dims[0]); - assert(ncomponents == r_dims[1]); + dataset_id = H5Dopen_ff(file_id, dataset_name, H5P_DEFAULT, rid1, + H5_EVENT_STACK_NULL); - num_points = H5Sget_select_elem_npoints(region_space); - assert(9 == num_points); + view_id = H5Vcreate_ff(dataset_id, query_id, H5P_DEFAULT, rid1, + H5_EVENT_STACK_NULL); + assert(view_id > 0); - ret = H5Sclose(region_space); - assert(0 == ret); - } + /* TODO do stuff here */ H5Vclose(view_id); - H5Qclose(query_id); - H5Qclose(query_id6); - H5Qclose(query_id5); - H5Qclose(query_id4); - H5Qclose(query_id3); - H5Qclose(query_id2); - H5Qclose(query_id1); - - /* Close the first dataset. */ ret = H5Dclose_ff(dataset_id, H5_EVENT_STACK_NULL); assert(0 == ret); - ret = H5Sclose(file_space_id); - assert(0 == ret); /* release container version 1. */ - ret = H5RCrelease(rid2, H5_EVENT_STACK_NULL); + ret = H5RCrelease(rid1, H5_EVENT_STACK_NULL); assert(0 == ret); ret = H5RCclose(rid1); assert(0 == ret); - ret = H5RCclose(rid2); - assert(0 == ret); - ret = H5TRclose(tid1); - assert(0 == ret); - MPI_Barrier(MPI_COMM_WORLD); + H5Qclose(query_id); + H5Qclose(query_id6); + H5Qclose(query_id5); + H5Qclose(query_id4); + H5Qclose(query_id3); + H5Qclose(query_id2); + H5Qclose(query_id1); - /* Close the file. */ - ret = H5Fclose_ff(file_id, H5_EVENT_STACK_NULL); - assert(0 == ret); + return query_id; } int @@ -247,6 +228,8 @@ main(int argc, char **argv) hsize_t ncomponents = 3; hsize_t start, total; int *data; + hid_t file_id, fapl_id; + herr_t ret; hsize_t i, j; int provided; @@ -277,11 +260,30 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); - write_dataset(file_name, dataset_name, total, ncomponents, H5T_NATIVE_INT, + /* Choose the IOD VOL plugin to use with this file. */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* Open an existing file. */ + file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, + H5_EVENT_STACK_NULL); + + ret = H5Pclose(fapl_id); + assert(0 == ret); + + write_dataset(file_id, dataset_name, total, ncomponents, H5T_NATIVE_INT, ntuples, start, data); MPI_Barrier(MPI_COMM_WORLD); + query_and_view(file_id, dataset_name); + + MPI_Barrier(MPI_COMM_WORLD); + + /* Close the file. */ + ret = H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL); + assert(0 == ret); + free(data); MPI_Barrier(MPI_COMM_WORLD); @@ -290,19 +292,3 @@ main(int argc, char **argv) return 0; } - -//H5Xregister(&index_plugin_struct); -//dsid = H5Dcreate(fid, "A", ...); -//H5Xcreate(fid, 32, dsid, H5P_DEFAULT); -//H5Dwrite(dsid, ...); -//H5Dclose(dsid); -// -//dsid = H5Dopen(fid, "A", ...); -//int query_val = 15; -//qid = H5Qcreate(H5Q_TYPE_DATA_ELEM, H5Q_MATCH_EQUAL, H5T_NATIVE_INT, &query_val); -//vid = H5Vcreate(dsid, qid, H5P_DEFAULT); -//H5Dclose(dsid); -//H5Xdelete(dsid, 32); - - - -- cgit v0.12