diff options
-rw-r--r-- | examples/h5ff_client_dset.c | 152 | ||||
-rw-r--r-- | examples/h5ff_client_map.c | 11 | ||||
-rw-r--r-- | src/H5VLiod.c | 94 | ||||
-rw-r--r-- | src/H5VLiod_dset.c | 28 |
4 files changed, 220 insertions, 65 deletions
diff --git a/examples/h5ff_client_dset.c b/examples/h5ff_client_dset.c index f571d74..5ae072c 100644 --- a/examples/h5ff_client_dset.c +++ b/examples/h5ff_client_dset.c @@ -31,9 +31,11 @@ int main(int argc, char **argv) { hsize_t dims[1]; hsize_t extent; + void *dset_token1, *dset_token2, *dset_token3; + size_t token_size1, token_size2, token_size3; int my_rank, my_size; int provided; - MPI_Request mpi_req; + MPI_Request mpi_req, mpi_reqs[6]; H5ES_status_t status; size_t num_events = 0; @@ -111,50 +113,100 @@ int main(int argc, char **argv) { Leader can tell its delegates that the transaction is started. */ if(0 == my_rank) { + trans_num = 1; ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_STACK_NULL); assert(0 == ret); - trans_num = 1; + /* Leader also create some objects in transaction 1 */ + + /* create group hierarchy /G1/G2/G3 */ + gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); + assert(gid1 > 0); + gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); + assert(gid2 > 0); + gid3 = H5Gcreate_ff(gid2, "G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); + assert(gid3 > 0); + + /* create datasets */ + did1 = H5Dcreate_ff(gid1, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); + assert(did1 > 0); + did2 = H5Dcreate_ff(gid2, "D2", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); + assert(did2 > 0); + did3 = H5Dcreate_ff(gid3, "D3", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); + assert(did3 > 0); + + ret = H5Gclose_ff(gid1, e_stack); + assert(ret == 0); + ret = H5Gclose_ff(gid2, e_stack); + assert(ret == 0); + ret = H5Gclose_ff(gid3, e_stack); + assert(ret == 0); } /* Tell Delegates that transaction 1 is started */ MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req); + /* Do the local-to-global, global-to-local, so all delegates can + write to the dsets created in transaction 1 */ + + if(0 == my_rank) { + /* get the token size of each dset */ + ret = H5Oget_token(did1, NULL, &token_size1); + assert(0 == ret); + ret = H5Oget_token(did2, NULL, &token_size2); + assert(0 == ret); + ret = H5Oget_token(did3, NULL, &token_size3); + assert(0 == ret); + + /* allocate buffers for each token */ + dset_token1 = malloc(token_size1); + dset_token2 = malloc(token_size2); + dset_token3 = malloc(token_size3); + + /* get the token buffer */ + ret = H5Oget_token(did1, dset_token1, &token_size1); + assert(0 == ret); + ret = H5Oget_token(did2, dset_token2, &token_size2); + assert(0 == ret); + ret = H5Oget_token(did3, dset_token3, &token_size3); + assert(0 == ret); + + /* bcast the token sizes and the tokens */ + MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); + MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]); + MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]); + MPI_Ibcast(dset_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]); + MPI_Ibcast(dset_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]); + MPI_Ibcast(dset_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]); + } + /* Leader can continue writing to transaction 1, while others wait for the ibcast to complete */ - if(0 != my_rank) + if(0 != my_rank) { MPI_Wait(&mpi_req, MPI_STATUS_IGNORE); - - assert(1 == trans_num); - - /* - Assume the following object create calls are collective. - - In a real world scenario, the following create calls are - independent by default; i.e. only 1 process, typically a leader - process, calls them. The user does the local-to-global, - global-to-local thing to get the objects accessible to all delegates. - - This will not fail here because IOD used for now is skeletal, - so it does not matter if the calls are collective or - independent. - */ - - /* create group hierarchy /G1/G2/G3 */ - gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); - assert(gid1 > 0); - gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); - assert(gid2 > 0); - gid3 = H5Gcreate_ff(gid2, "G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); - assert(gid3 > 0); - - /* create datasets */ - did1 = H5Dcreate_ff(gid1, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); - assert(did1 > 0); - did2 = H5Dcreate_ff(gid2, "D2", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); - assert(did2 > 0); - did3 = H5Dcreate_ff(gid3, "D3", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack); - assert(did3 > 0); + assert(1 == trans_num); + + /* recieve the token sizes */ + MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); + MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]); + MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]); + MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE); + + /* allocate buffers for each token */ + dset_token1 = malloc(token_size1); + dset_token2 = malloc(token_size2); + dset_token3 = malloc(token_size3); + + /* recieve the tokens */ + MPI_Ibcast(dset_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]); + MPI_Ibcast(dset_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]); + MPI_Ibcast(dset_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]); + MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE); + + did1 = H5Oopen_by_token(dset_token1, rid1, e_stack); + did2 = H5Oopen_by_token(dset_token2, rid1, e_stack); + did3 = H5Oopen_by_token(dset_token3, rid1, e_stack); + } /* write data to datasets */ @@ -228,8 +280,6 @@ int main(int argc, char **argv) { /* close some objects */ ret = H5Dclose_ff(did1, e_stack); assert(ret == 0); - ret = H5Gclose_ff(gid1, e_stack); - assert(ret == 0); /* release container version 0. This is async. */ ret = H5RCrelease(rid1, e_stack); @@ -341,17 +391,25 @@ int main(int argc, char **argv) { /* Do more updates on transaction 2 */ - extent = 10; - ret = H5Dset_extent_ff(did1, &extent, tid2, e_stack); - assert(ret == 0); + if(0 == my_rank) { + extent = 10; + ret = H5Dset_extent_ff(did1, &extent, tid2, e_stack); + assert(ret == 0); + } - extent = 30; - ret = H5Dset_extent_ff(did2, &extent, tid2, e_stack); - assert(ret == 0); + if((my_size > 1 && 1 == my_rank) || + (my_size == 1 && 0 == my_rank)) { + extent = 30; + ret = H5Dset_extent_ff(did2, &extent, tid2, e_stack); + assert(ret == 0); + } - extent = 60; - ret = H5Dset_extent_ff(did3, &extent, tid2, e_stack); - assert(ret == 0); + if((my_size > 2 && 2 == my_rank) || + (my_size == 1 && 0 == my_rank)) { + extent = 60; + ret = H5Dset_extent_ff(did3, &extent, tid2, e_stack); + assert(ret == 0); + } /* finish transaction 2 - all have to call */ ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, e_stack); @@ -372,10 +430,6 @@ int main(int argc, char **argv) { assert(ret == 0); ret = H5Gclose_ff(gid1, e_stack); assert(ret == 0); - ret = H5Gclose_ff(gid2, e_stack); - assert(ret == 0); - ret = H5Gclose_ff(gid3, e_stack); - assert(ret == 0); ret = H5Sclose(sid); assert(ret == 0); diff --git a/examples/h5ff_client_map.c b/examples/h5ff_client_map.c index f7e2557..4b2c8f9 100644 --- a/examples/h5ff_client_map.c +++ b/examples/h5ff_client_map.c @@ -319,10 +319,13 @@ int main(int argc, char **argv) { ret = H5Pclose(trspl_id);
assert(0 == ret);
- /* modify container contents using transaction started. */
- key = 1;
- ret = H5Mdelete_ff(map3, H5T_STD_I32LE, &key, tid2, e_stack);
- assert(ret == 0);
+ if((my_size > 1 && 1 == my_rank) ||
+ (my_size == 1 && 0 == my_rank)) {
+ /* modify container contents using transaction started. */
+ key = 1;
+ ret = H5Mdelete_ff(map3, H5T_STD_I32LE, &key, tid2, e_stack);
+ assert(ret == 0);
+ }
/* finish transaction 2 */
ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, e_stack);
diff --git a/src/H5VLiod.c b/src/H5VLiod.c index b1dcd3a..8708cad 100644 --- a/src/H5VLiod.c +++ b/src/H5VLiod.c @@ -2281,6 +2281,25 @@ H5VL_iod_dataset_create(void *_obj, H5VL_loc_params_t UNUSED loc_params, if(H5P_get(plist, H5VL_DSET_LCPL_ID, &lcpl_id) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get property value for lcpl id"); + /* Check that no values other than the first dimension in MAX dims + is H5S_UNLIMITED. */ + { + H5S_t *ds = NULL; + int ndims, i; + hsize_t max_dims[H5S_MAX_RANK]; + + if(NULL == (ds = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a dataspace") + + ndims = (int)H5S_GET_EXTENT_NDIMS(ds); + H5S_get_simple_extent_dims(ds, NULL, max_dims); + + for(i=1; i<ndims; i++) { + if(max_dims[i] == H5S_UNLIMITED) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "only first dimension can be H5S_UNLIMITED."); + } + } + /* get the transaction ID */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(dxpl_id))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, NULL, "can't find object for ID"); @@ -2567,6 +2586,36 @@ H5VL_iod_dataset_read(void *_dset, hid_t mem_type_id, hid_t mem_space_id, dset->common.request = NULL; } + /* At the IOD server, the array object is actually created with + MAX dims, not current dims. So here we need to range check the + filespace against the current dimensions of the dataset to make + sure that the I/O happens in the current range and not the + extensible one. */ + { + H5S_t *dset_space, *io_space; + hsize_t dset_dims[H5S_MAX_RANK], io_dims[H5S_MAX_RANK]; + int dset_ndims, io_ndims, i; + + if(NULL == (dset_space = (H5S_t *)H5I_object_verify(dset->remote_dset.space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") + if(NULL == (io_space = (H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") + + dset_ndims = (int)H5S_GET_EXTENT_NDIMS(dset_space); + io_ndims = (int)H5S_GET_EXTENT_NDIMS(io_space); + + if(dset_ndims < io_ndims) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection not within dataset's dataspace"); + + H5S_get_simple_extent_dims(dset_space, dset_dims, NULL); + H5S_get_simple_extent_dims(io_space, io_dims, NULL); + + for(i=0 ; i<io_ndims ; i++) { + if(dset_dims[i] < io_dims[i]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection Out of range"); + } + } + /* check arguments */ if(H5S_ALL != mem_space_id) { if(NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) @@ -2768,6 +2817,36 @@ H5VL_iod_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id, dset->common.request = NULL; } + /* At the IOD server, the array object is actually created with + MAX dims, not current dims. So here we need to range check the + filespace against the current dimensions of the dataset to make + sure that the I/O happens in the current range and not the + extensible one. */ + { + H5S_t *dset_space, *io_space; + hsize_t dset_dims[H5S_MAX_RANK], io_dims[H5S_MAX_RANK]; + int dset_ndims, io_ndims, i; + + if(NULL == (dset_space = (H5S_t *)H5I_object_verify(dset->remote_dset.space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") + if(NULL == (io_space = (H5S_t *)H5I_object_verify(file_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") + + dset_ndims = (int)H5S_GET_EXTENT_NDIMS(dset_space); + io_ndims = (int)H5S_GET_EXTENT_NDIMS(io_space); + + if(dset_ndims < io_ndims) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection not within dataset's dataspace"); + + H5S_get_simple_extent_dims(dset_space, dset_dims, NULL); + H5S_get_simple_extent_dims(io_space, io_dims, NULL); + + for(i=0 ; i<io_ndims ; i++) { + if(dset_dims[i] < io_dims[i]) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "selection Out of range"); + } + } + /* check arguments */ if(H5S_ALL != mem_space_id) { if(NULL == (mem_space = (const H5S_t *)H5I_object_verify(mem_space_id, H5I_DATASPACE))) @@ -5162,6 +5241,11 @@ H5VL_iod_obj_open_token(const void *token, H5RC_t *rc, H5I_type_t *opened_type, dset->common.file->nopen_objs ++; dset->common.obj_name = NULL; +#if H5VL_IOD_DEBUG + printf("Dataset open by token %llu: ID %llu\n", + g_axe_id, input.iod_id); +#endif + if(H5VL__iod_create_and_forward(H5VL_OBJECT_OPEN_BY_TOKEN_ID, HG_OBJECT_OPEN_BY_TOKEN, (H5VL_iod_object_t *)dset, 1, 0, NULL, (H5VL_iod_req_info_t *)rc, &input, @@ -5214,6 +5298,11 @@ H5VL_iod_obj_open_token(const void *token, H5RC_t *rc, H5I_type_t *opened_type, dtype->common.file->nopen_objs ++; dtype->common.obj_name = NULL; +#if H5VL_IOD_DEBUG + printf("Named Datatype open by token %llu: ID %llu\n", + g_axe_id, input.iod_id); +#endif + if(H5VL__iod_create_and_forward(H5VL_OBJECT_OPEN_BY_TOKEN_ID, HG_OBJECT_OPEN_BY_TOKEN, (H5VL_iod_object_t *)dtype, 1, 0, NULL, (H5VL_iod_req_info_t *)rc, &input, @@ -5250,6 +5339,11 @@ H5VL_iod_obj_open_token(const void *token, H5RC_t *rc, H5I_type_t *opened_type, grp->common.file->nopen_objs ++; grp->common.obj_name = NULL; +#if H5VL_IOD_DEBUG + printf("Group open by token %llu: ID %llu\n", + g_axe_id, input.iod_id); +#endif + if(H5VL__iod_create_and_forward(H5VL_OBJECT_OPEN_BY_TOKEN_ID, HG_OBJECT_OPEN_BY_TOKEN, (H5VL_iod_object_t *)grp, 1, 0, NULL, (H5VL_iod_req_info_t *)rc, &input, diff --git a/src/H5VLiod_dset.c b/src/H5VLiod_dset.c index d558fa7..38235d3 100644 --- a/src/H5VLiod_dset.c +++ b/src/H5VLiod_dset.c @@ -92,15 +92,16 @@ H5VL_iod_server_dset_create_cb(AXE_engine_t UNUSED axe_engine, iod_trans_id_t wtid = input->trans_num; iod_trans_id_t rtid = input->rcxt_num; uint32_t cs_scope = input->cs_scope; + hid_t space_id = input->space_id; iod_handle_t dset_oh, cur_oh, mdkv_oh; iod_obj_id_t cur_id; const char *name = input->name; /* name of dset including path to create */ char *last_comp; /* the name of the dataset obtained from the last component in the path */ hid_t dcpl_id; iod_array_struct_t array; /* IOD array struct describing the dataset's dimensions */ - iod_size_t *max_dims; scratch_pad sp; iod_ret_t ret; + iod_size_t array_dims[H5S_MAX_RANK], current_dims[H5S_MAX_RANK]; hbool_t collective = FALSE; /* MSC - change when we allow for collective */ herr_t ret_value = SUCCEED; @@ -115,14 +116,19 @@ H5VL_iod_server_dset_create_cb(AXE_engine_t UNUSED axe_engine, /* Set the IOD array creation parameters */ array.cell_size = H5Tget_size(input->type_id); - array.num_dims = H5Sget_simple_extent_ndims(input->space_id); - if(NULL == (array.current_dims = (iod_size_t *)malloc (sizeof(iod_size_t) * array.num_dims))) - HGOTO_ERROR(H5E_SYM, H5E_NOSPACE, FAIL, "can't allocate dimention size array"); - if(NULL == (max_dims = (iod_size_t *)malloc (sizeof(iod_size_t) * array.num_dims))) - HGOTO_ERROR(H5E_SYM, H5E_NOSPACE, FAIL, "can't allocate dimention size array"); - if(H5Sget_simple_extent_dims(input->space_id, array.current_dims, max_dims) < 0) + array.num_dims = H5Sget_simple_extent_ndims(space_id); + + if(H5Sget_simple_extent_dims(space_id, current_dims, array_dims) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't get dimentions' sizes"); - array.firstdim_max = max_dims[0]; + + if(H5S_UNLIMITED == array_dims[0]) { + array_dims[0] = current_dims[0]; + array.firstdim_max = H5S_UNLIMITED; + } + else { + array.firstdim_max = array_dims[0]; + } + array.chunk_dims = NULL; array.dims_seq = NULL; @@ -215,7 +221,7 @@ H5VL_iod_server_dset_create_cb(AXE_engine_t UNUSED axe_engine, HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't insert KV value"); /* insert dataspace metadata */ - if(H5VL_iod_insert_dataspace(mdkv_oh, wtid, input->space_id, + if(H5VL_iod_insert_dataspace(mdkv_oh, wtid, space_id, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't insert KV value"); @@ -238,7 +244,7 @@ H5VL_iod_server_dset_create_cb(AXE_engine_t UNUSED axe_engine, #if H5_DO_NATIVE H5Pset_alloc_time(input->dcpl_id,H5D_ALLOC_TIME_EARLY); cur_oh.cookie = H5Dcreate2(loc_handle.cookie, last_comp, input->type_id, - input->space_id, input->lcpl_id, + space_id, input->lcpl_id, input->dcpl_id, input->dapl_id); assert(cur_oh.cookie); #endif @@ -259,8 +265,6 @@ done: HG_Handler_start_output(op_data->hg_handle, &output); } - if(max_dims) free(max_dims); - if(array.current_dims) free(array.current_dims); last_comp = (char *)H5MM_xfree(last_comp); input = (dset_create_in_t *)H5MM_xfree(input); op_data = (op_data_t *)H5MM_xfree(op_data); |