diff options
author | M. Scot Breitenfeld <brtnfld@hdfgroup.org> | 2017-10-26 15:13:33 (GMT) |
---|---|---|
committer | M. Scot Breitenfeld <brtnfld@hdfgroup.org> | 2017-10-26 15:13:33 (GMT) |
commit | 59e7c7703e62717e7bf482bec2bbad52aada3c80 (patch) | |
tree | 59ebe294f77f57850a017ff9596234066d3acebd /src | |
parent | c3aa3978b2873b158a1917f8961c035f413b5e56 (diff) | |
parent | 084704392fa93d1569861c53504e89236468a3b6 (diff) | |
download | hdf5-59e7c7703e62717e7bf482bec2bbad52aada3c80.zip hdf5-59e7c7703e62717e7bf482bec2bbad52aada3c80.tar.gz hdf5-59e7c7703e62717e7bf482bec2bbad52aada3c80.tar.bz2 |
Merge branch 'develop' into hdf5_1_10
Diffstat (limited to 'src')
48 files changed, 3829 insertions, 1039 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 178c954..96ea589 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -516,6 +516,9 @@ IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" ) set (H5PL_SOURCES ${HDF5_SRC_DIR}/H5PL.c + ${HDF5_SRC_DIR}/H5PLint.c + ${HDF5_SRC_DIR}/H5PLpath.c + ${HDF5_SRC_DIR}/H5PLplugin_cache.c ) set (H5PL_HDRS @@ -29,6 +29,7 @@ #include "H5Pprivate.h" /* Property lists */ #include "H5SLprivate.h" /* Skip lists */ #include "H5Tprivate.h" /* Datatypes */ +#include "H5FSprivate.h" /* File free space */ /****************/ /* Local Macros */ @@ -204,6 +205,10 @@ H5_init_library(void) * property classes. * The link interface needs to be initialized so that link property lists * have their properties registered. + * The FS module needs to be initialized as a result of the fix for HDFFV-10160: + * It might not be initialized during normal file open. + * When the application does not close the file, routines in the module might + * be called via H5_term_library() when shutting down the file. */ if(H5E_init() < 0) HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize error interface") @@ -217,6 +222,8 @@ H5_init_library(void) HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize metadata caching interface") if(H5L_init() < 0) HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize link interface") + if(H5FS_init() < 0) + HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize FS interface") /* Debugging? */ H5_debug_mask("-all"); diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index b9e2a60..083ee5b 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -230,7 +230,7 @@ typedef struct H5AC_proxy_entry_t { /* (Note that this currently duplicates some cache functionality) */ } H5AC_proxy_entry_t; - +/* Name of property for ring info in DXPL */ #define H5AC_RING_NAME "H5AC_ring_type" /* Dataset transfer property lists for metadata calls */ @@ -247,7 +247,6 @@ H5_DLLVAR hid_t H5AC_noio_dxpl_id; H5_DLLVAR hid_t H5AC_rawdata_dxpl_id; /* Default cache configuration. */ - #define H5AC__DEFAULT_METADATA_WRITE_STRATEGY \ H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED @@ -366,6 +366,7 @@ H5C_create(size_t max_cache_size, cache_ptr->coll_write_list = NULL; #endif /* H5_HAVE_PARALLEL */ +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS cache_ptr->cLRU_list_len = 0; cache_ptr->cLRU_list_size = (size_t)0; cache_ptr->cLRU_head_ptr = NULL; @@ -375,6 +376,7 @@ H5C_create(size_t max_cache_size, cache_ptr->dLRU_list_size = (size_t)0; cache_ptr->dLRU_head_ptr = NULL; cache_ptr->dLRU_tail_ptr = NULL; +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ cache_ptr->size_increase_possible = FALSE; cache_ptr->flash_size_increase_possible = FALSE; @@ -1481,8 +1483,10 @@ H5C_insert_entry(H5F_t * f, entry_ptr->next = NULL; entry_ptr->prev = NULL; +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS entry_ptr->aux_next = NULL; entry_ptr->aux_prev = NULL; +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ #ifdef H5_HAVE_PARALLEL entry_ptr->coll_next = NULL; @@ -4062,8 +4066,7 @@ H5C_destroy_flush_dependency(void *parent_thing, void * child_thing) child_entry->flush_dep_parent_nalloc = 0; } /* end if */ else if(child_entry->flush_dep_parent_nalloc > H5C_FLUSH_DEP_PARENT_INIT - && child_entry->flush_dep_nparents - <= (child_entry->flush_dep_parent_nalloc / 4)) { + && child_entry->flush_dep_nparents <= (child_entry->flush_dep_parent_nalloc / 4)) { if(NULL == (child_entry->flush_dep_parent = (H5C_cache_entry_t **)H5FL_BLK_REALLOC(parent, child_entry->flush_dep_parent, (child_entry->flush_dep_parent_nalloc / 4) * sizeof(H5C_cache_entry_t *)))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for flush dependency parent list") child_entry->flush_dep_parent_nalloc /= 4; @@ -6273,17 +6276,27 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to insert skip list item") } /* end if */ else + { #endif /* H5_HAVE_PARALLEL */ - if(entry_ptr->prefetched) { - HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID); - mem_type = cache_ptr->class_table_ptr[entry_ptr->prefetch_type_id]->mem_type; - } /* end if */ - else - mem_type = entry_ptr->type->mem_type; + if(entry_ptr->prefetched) { + HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID); + mem_type = cache_ptr-> + class_table_ptr[entry_ptr->prefetch_type_id]-> + mem_type; + } /* end if */ + else + mem_type = entry_ptr->type->mem_type; + + if(H5F_block_write(f, mem_type, entry_ptr->addr, + entry_ptr->size, dxpl_id, + entry_ptr->image_ptr) < 0) - if(H5F_block_write(f, mem_type, entry_ptr->addr, entry_ptr->size, dxpl_id, entry_ptr->image_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file") + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ + "Can't write image to file") +#ifdef H5_HAVE_PARALLEL + } +#endif /* H5_HAVE_PARALLEL */ } /* end if */ /* if the entry has a notify callback, notify it that we have @@ -6418,7 +6431,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, HDassert(entry_ptr->flush_dep_ndirty_children == 0); if(entry_ptr->flush_dep_nparents > 0) if(H5C__mark_flush_dep_clean(entry_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag") + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "Can't propagate flush dep clean flag") } /* end if */ } /* end else */ @@ -6940,8 +6953,10 @@ H5C_load_entry(H5F_t * f, entry->next = NULL; entry->prev = NULL; +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS entry->aux_next = NULL; entry->aux_prev = NULL; +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ #ifdef H5_HAVE_PARALLEL entry->coll_next = NULL; @@ -7276,6 +7291,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, HDassert( H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS ); +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS initial_list_len = cache_ptr->cLRU_list_len; entry_ptr = cache_ptr->cLRU_tail_ptr; @@ -7320,6 +7336,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, entry_ptr = prev_ptr; entries_examined++; } +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ } done: @@ -7979,7 +7996,7 @@ done: static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry) { - unsigned u; /* Local index variable */ + int i; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -7988,16 +8005,19 @@ H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry) HDassert(entry); /* Iterate over the parent entries, if any */ - for(u = 0; u < entry->flush_dep_nparents; u++) { + /* Note reverse iteration order, in case the callback removes the flush + * dependency - QAK, 2017/08/12 + */ + for(i = ((int)entry->flush_dep_nparents) - 1; i >= 0; i--) { /* Sanity check */ - HDassert(entry->flush_dep_parent[u]->flush_dep_ndirty_children > 0); + HDassert(entry->flush_dep_parent[i]->flush_dep_ndirty_children > 0); /* Adjust the parent's number of dirty children */ - entry->flush_dep_parent[u]->flush_dep_ndirty_children--; + entry->flush_dep_parent[i]->flush_dep_ndirty_children--; /* If the parent has a 'notify' callback, send a 'child entry cleaned' notice */ - if(entry->flush_dep_parent[u]->type->notify && - (entry->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, entry->flush_dep_parent[u]) < 0) + if(entry->flush_dep_parent[i]->type->notify && + (entry->flush_dep_parent[i]->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, entry->flush_dep_parent[i]) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry dirty flag reset") } /* end for */ @@ -8023,7 +8043,7 @@ done: herr_t H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr) { - unsigned u; /* Local index variable */ + int i; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -8032,18 +8052,21 @@ H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr) HDassert(entry_ptr); /* Iterate over the parent entries, if any */ - for(u = 0; u < entry_ptr->flush_dep_nparents; u++) { - + /* Note reverse iteration order, in case the callback removes the flush + * dependency - QAK, 2017/08/12 + */ + for(i = ((int)entry_ptr->flush_dep_nparents) - 1; i >= 0; i--) { + /* Sanity checks */ HDassert(entry_ptr->flush_dep_parent); - HDassert(entry_ptr->flush_dep_parent[u]->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(entry_ptr->flush_dep_parent[u]->flush_dep_nunser_children > 0); + HDassert(entry_ptr->flush_dep_parent[i]->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(entry_ptr->flush_dep_parent[i]->flush_dep_nunser_children > 0); /* decrement the parents number of unserialized children */ - entry_ptr->flush_dep_parent[u]->flush_dep_nunser_children--; + entry_ptr->flush_dep_parent[i]->flush_dep_nunser_children--; /* If the parent has a 'notify' callback, send a 'child entry serialized' notice */ - if(entry_ptr->flush_dep_parent[u]->type->notify && - (entry_ptr->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_SERIALIZED, entry_ptr->flush_dep_parent[u]) < 0) + if(entry_ptr->flush_dep_parent[i]->type->notify && + (entry_ptr->flush_dep_parent[i]->type->notify)(H5C_NOTIFY_ACTION_CHILD_SERIALIZED, entry_ptr->flush_dep_parent[i]) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry serialized flag set") } /* end for */ @@ -8055,7 +8078,7 @@ done: /*------------------------------------------------------------------------- * Function: H5C__mark_flush_dep_unserialized() * - * Purpose: Decrement the flush_dep_nunser_children fields of all the + * Purpose: Increment the flush_dep_nunser_children fields of all the * target entry's flush dependency parents in response to * the target entry becoming unserialized. * @@ -8862,6 +8885,12 @@ H5C_remove_entry(void *_entry) HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove protected entry from cache") if(entry->is_pinned) HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove pinned entry from cache") + /* NOTE: If these two errors are getting tripped because the entry is + * in a flush dependency with a freedspace entry, move the checks + * after the "before evict" message is sent, and add the + * "child being evicted" message to the "before evict" notify + * section below. QAK - 2017/08/03 + */ if(entry->flush_dep_nparents > 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry with flush dependency parents from cache") if(entry->flush_dep_nchildren > 0) diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index 4a08d9b..08c70d9 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -716,6 +716,7 @@ H5C_stats(H5C_t * cache_ptr, (long)(cache_ptr->LRU_list_size), (unsigned long)(cache_ptr->LRU_list_len)); +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS HDfprintf(stdout, "%s current clean LRU size / length = %ld / %lu\n", cache_ptr->prefix, @@ -727,6 +728,7 @@ H5C_stats(H5C_t * cache_ptr, cache_ptr->prefix, (long)(cache_ptr->dLRU_list_size), (unsigned long)(cache_ptr->dLRU_list_len)); +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ HDfprintf(stdout, "%s Total hits / misses / hit_rate = %ld / %ld / %f\n", diff --git a/src/H5Cimage.c b/src/H5Cimage.c index debd30c..53d1712 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -649,8 +649,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, /* Initialize fields supporting replacement policies: */ ds_entry_ptr->next = NULL; ds_entry_ptr->prev = NULL; +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS ds_entry_ptr->aux_next = NULL; ds_entry_ptr->aux_prev = NULL; +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ #ifdef H5_HAVE_PARALLEL pf_entry_ptr->coll_next = NULL; pf_entry_ptr->coll_prev = NULL; diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index fdb14a5..d431887 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -4746,6 +4746,7 @@ struct H5C_t { H5C_cache_entry_t * LRU_head_ptr; H5C_cache_entry_t * LRU_tail_ptr; +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS /* Fields for clean LRU list of entries */ uint32_t cLRU_list_len; size_t cLRU_list_size; @@ -4757,6 +4758,7 @@ struct H5C_t { size_t dLRU_list_size; H5C_cache_entry_t * dLRU_head_ptr; H5C_cache_entry_t * dLRU_tail_ptr; +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ #ifdef H5_HAVE_PARALLEL /* Fields for collective metadata reads */ diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 5335f80..50732ca 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -1648,8 +1648,10 @@ typedef struct H5C_cache_entry_t { /* fields supporting replacement policies: */ struct H5C_cache_entry_t *next; struct H5C_cache_entry_t *prev; +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS struct H5C_cache_entry_t *aux_next; struct H5C_cache_entry_t *aux_prev; +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ #ifdef H5_HAVE_PARALLEL struct H5C_cache_entry_t *coll_next; struct H5C_cache_entry_t *coll_prev; diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index b7b8b03..af6599a 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -299,9 +299,6 @@ static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info, static herr_t H5D__chunk_cache_prune(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t *dxpl_cache, size_t size); static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk); -static herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, - const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert, - hsize_t scaled[]); #ifdef H5_HAVE_PARALLEL static herr_t H5D__chunk_collective_fill(const H5D_t *dset, hid_t dxpl_id, H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, const void *fill_buf); @@ -6558,7 +6555,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert, hsize_t scaled[]) { diff --git a/src/H5Defl.c b/src/H5Defl.c index 5536ba3..ebe7689 100644 --- a/src/H5Defl.c +++ b/src/H5Defl.c @@ -288,7 +288,7 @@ H5D__efl_read(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed") if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0) HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name") - if((fd = HDopen(full_name, O_RDONLY, 0)) < 0) + if((fd = HDopen(full_name, O_RDONLY)) < 0) HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file") if(HDlseek(fd, (HDoff_t)(efl->slot[u].offset + (HDoff_t)skip), SEEK_SET) < 0) HGOTO_ERROR(H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file") @@ -380,7 +380,7 @@ H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t siz HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed") if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0) HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name") - if((fd = HDopen(full_name, O_CREAT | O_RDWR, 0666)) < 0) { + if((fd = HDopen(full_name, O_CREAT | O_RDWR, H5_POSIX_CREATE_MODE_RW)) < 0) { if(HDaccess(full_name, F_OK) < 0) HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "external raw data file does not exist") else diff --git a/src/H5Dint.c b/src/H5Dint.c index 3b938e2..bdedd1e 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1213,10 +1213,6 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, /* Don't allow compact datasets to allocate space later */ if(layout->type == H5D_COMPACT && fill->alloc_time != H5D_ALLOC_TIME_EARLY) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "compact dataset must have early space allocation") - - /* If MPI VFD is used, no filter support yet. */ - if(H5F_HAS_FEATURE(file, H5FD_FEAT_HAS_MPI) && pline->nused > 0) - HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, NULL, "Parallel I/O does not support filters yet") } /* end if */ /* Set the latest version of the layout, pline & fill messages, if requested */ diff --git a/src/H5Dio.c b/src/H5Dio.c index 1766422..104a632 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -714,11 +714,6 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, if(H5T_get_class(type_info.mem_type, TRUE) == H5T_REFERENCE && H5T_get_ref_type(type_info.mem_type) == H5R_DATASET_REGION) HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "Parallel IO does not support writing region reference datatypes yet") - - /* Can't write to chunked datasets with filters, in parallel */ - if(dataset->shared->layout.type == H5D_CHUNKED && - dataset->shared->dcpl_cache.pline.nused > 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot write to chunked storage with filters in parallel") } /* end if */ else { /* Collective access is not permissible without a MPI based VFD */ @@ -1195,7 +1190,7 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hid_t dxpl_id, HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator") /* Check if we can set direct MPI-IO read/write functions */ - if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, fm, dx_plist)) < 0) + if((opt = H5D__mpio_opt_possible(io_info, file_space, mem_space, type_info, dx_plist)) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for direct IO dataspace ") /* Check if we can use the optimized parallel I/O routines */ @@ -1207,6 +1202,67 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, hid_t dxpl_id, io_info->io_ops.single_write = H5D__mpio_select_write; } /* end if */ else { + /* Check if there are any filters in the pipeline. If there are, + * we cannot break to independent I/O if this is a write operation; + * otherwise there will be metadata inconsistencies in the file. + */ + if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) { + H5D_mpio_no_collective_cause_t cause; + uint32_t local_no_collective_cause; + uint32_t global_no_collective_cause; + hbool_t local_error_message_previously_written = FALSE; + hbool_t global_error_message_previously_written = FALSE; + size_t index; + char local_no_collective_cause_string[256] = ""; + char global_no_collective_cause_string[256] = ""; + const char *cause_strings[] = { "independent I/O was requested", + "datatype conversions were required", + "data transforms needed to be applied", + "optimized MPI types flag wasn't set", + "one of the dataspaces was neither simple nor scalar", + "dataset was not contiguous or chunked" }; + + if (H5P_get(dx_plist, H5D_MPIO_LOCAL_NO_COLLECTIVE_CAUSE_NAME, &local_no_collective_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get local no collective cause value") + if (H5P_get(dx_plist, H5D_MPIO_GLOBAL_NO_COLLECTIVE_CAUSE_NAME, &global_no_collective_cause) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get global no collective cause value") + + /* Append each of the "reason for breaking collective I/O" error messages to the + * local and global no collective cause strings */ + for (cause = 1, index = 0; cause < H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE; cause <<= 1, index++) { + size_t cause_strlen = strlen(cause_strings[index]); + + if (cause & local_no_collective_cause) { + /* Check if there were any previous error messages included. If so, prepend a semicolon + * to separate the messages. + */ + if (local_error_message_previously_written) strncat(local_no_collective_cause_string, "; ", 2); + + strncat(local_no_collective_cause_string, cause_strings[index], cause_strlen); + + local_error_message_previously_written = TRUE; + } /* end if */ + + if (cause & global_no_collective_cause) { + /* Check if there were any previous error messages included. If so, prepend a semicolon + * to separate the messages. + */ + if (global_error_message_previously_written) strncat(global_no_collective_cause_string, "; ", 2); + + strncat(global_no_collective_cause_string, cause_strings[index], cause_strlen); + + global_error_message_previously_written = TRUE; + } /* end if */ + } /* end for */ + + HGOTO_ERROR(H5E_IO, H5E_NO_INDEPENDENT, FAIL, "Can't perform independent write with filters in pipeline.\n" + " The following caused a break from collective I/O:\n" + " Local causes: %s\n" + " Global causes: %s", + local_no_collective_cause_string, + global_no_collective_cause_string); + } /* end if */ + /* If we won't be doing collective I/O, but the user asked for * collective I/O, change the request to use independent I/O, but * mark it so that we remember to revert the change. diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 0389c72..79572c0 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -84,7 +84,6 @@ #define H5D_CHUNK_SELECT_IRREG 2 #define H5D_CHUNK_SELECT_NONE 0 - /******************/ /* Local Typedefs */ /******************/ @@ -94,6 +93,113 @@ typedef struct H5D_chunk_addr_info_t { H5D_chunk_info_t chunk_info; } H5D_chunk_addr_info_t; +/* + * Information about a single chunk when performing collective filtered I/O. All + * of the fields of one of these structs are initialized at the start of collective + * filtered I/O in the function H5D__construct_filtered_io_info_list(). + * + * This struct's fields are as follows: + * + * index - The "Index" of the chunk in the dataset. The index of a chunk is used during + * the collective re-insertion of chunks into the chunk index after the collective + * I/O has been performed. + * + * scaled - The scaled coordinates of the chunk in the dataset's file dataspace. The + * coordinates are used in both the collective re-allocation of space in the file + * and the collective re-insertion of chunks into the chunk index after the collective + * I/O has been performed. + * + * full_overwrite - A flag which determines whether or not a chunk needs to be read from the + * file when being updated. If a chunk is being fully overwritten (the entire + * extent is selected in its file dataspace), then it is not necessary to + * read the chunk from the file. However, if the chunk is not being fully + * overwritten, it has to be read from the file in order to update the chunk + * without trashing the parts of the chunk that are not selected. + * + * num_writers - The total number of processors writing to this chunk. This field is used + * when the new owner of a chunk is receiving messages, which contain selections in + * the chunk and data to update the chunk with, from other processors which have this + * chunk selected in the I/O operation. The new owner must know how many processors it + * should expect messages from so that it can post an equal number of receive calls. + * + * io_size - The total size of I/O to this chunk. This field is an accumulation of the size of + * I/O to the chunk from each processor which has the chunk selected and is used to + * determine the value for the previous full_overwrite flag. + * + * buf - A pointer which serves the dual purpose of holding either the chunk data which is to be + * written to the file or the chunk data which has been read from the file. + * + * chunk_states - In the case of dataset writes only, this struct is used to track a chunk's size and + * address in the file before and after the filtering operation has occurred. + * + * Its fields are as follows: + * + * chunk_current - The address in the file and size of this chunk before the filtering + * operation. When reading a chunk from the file, this field is used to + * read the correct amount of bytes. It is also used when redistributing + * shared chunks among processors and as a parameter to the chunk file + * space reallocation function. + * + * new_chunk - The address in the file and size of this chunk after the filtering + * operation. This field is relevant when collectively re-allocating space + * in the file for all of the chunks written to in the I/O operation, as + * their sizes may have changed after their data has been filtered. + * + * owners - In the case of dataset writes only, this struct is used to manage which single processor + * will ultimately write data out to the chunk. It allows the other processors to act according + * to the decision and send their selection in the chunk, as well as the data they wish + * to update the chunk with, to the processor which is writing to the chunk. + * + * Its fields are as follows: + * + * original_owner - The processor which originally had this chunk selected at the beginning of + * the collective filtered I/O operation. This field is currently used when + * redistributing shared chunks among processors. + * + * new_owner - The processor which has been selected to perform the write to this chunk. + * + * async_info - In the case of dataset writes only, this struct is used by the owning processor of the + * chunk in order to manage the MPI send and receive calls made between it and all of + * the other processors which have this chunk selected in the I/O operation. + * + * Its fields are as follows: + * + * receive_requests_array - An array containing one MPI_Request for each of the + * asynchronous MPI receive calls the owning processor of this + * chunk makes to another processor in order to receive that + * processor's chunk modification data and selection in the chunk. + * + * receive_buffer_array - An array of buffers into which the owning processor of this chunk + * will store chunk modification data and the selection in the chunk + * received from another processor. + * + * num_receive_requests - The number of entries in the receive_request_array and + * receive_buffer_array fields. + */ +typedef struct H5D_filtered_collective_io_info_t { + hsize_t index; + hsize_t scaled[H5O_LAYOUT_NDIMS]; + hbool_t full_overwrite; + size_t num_writers; + size_t io_size; + void *buf; + + struct { + H5F_block_t chunk_current; + H5F_block_t new_chunk; + } chunk_states; + + struct { + int original_owner; + int new_owner; + } owners; + + struct { + MPI_Request *receive_requests_array; + unsigned char **receive_buffer_array; + int num_receive_requests; + } async_info; +} H5D_filtered_collective_io_info_t; /********************/ /* Local Prototypes */ @@ -103,9 +209,15 @@ static herr_t H5D__chunk_collective_io(H5D_io_info_t *io_info, static herr_t H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist); +static herr_t H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, + H5P_genplist_t *dx_plist); static herr_t H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, int sum_chunk, H5P_genplist_t *dx_plist); +static herr_t H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, + H5P_genplist_t *dx_plist); static herr_t H5D__inter_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5S_t *file_space, const H5S_t *mem_space); @@ -124,6 +236,26 @@ static herr_t H5D__mpio_get_min_chunk(const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, int *min_chunkf); static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, int *sum_chunkf); +static herr_t H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm, + H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries); +static herr_t H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm, + H5D_filtered_collective_io_info_t *local_chunk_array, size_t *local_chunk_array_num_entries); +static herr_t H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries, + size_t array_entry_size, void **gathered_array, size_t *gathered_array_num_entries, + int nprocs, hbool_t allgather, int root, MPI_Comm comm, int (*sort_func)(const void *, const void *)); +static herr_t H5D__mpio_filtered_collective_write_type( + H5D_filtered_collective_io_info_t *chunk_list, size_t num_entries, + MPI_Datatype *new_mem_type, hbool_t *mem_type_derived, + MPI_Datatype *new_file_type, hbool_t *file_type_derived); +static herr_t H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry, + const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm); +static int H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2); +static int H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, + const void *filtered_collective_io_info_entry2); +static int H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, + const void *filtered_collective_io_info_entry2); /*********************/ @@ -142,7 +274,7 @@ static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info, * Purpose: Checks if an direct I/O transfer is possible between memory and * the file. * - * Return: Sauccess: Non-negative: TRUE or FALSE + * Return: Success: Non-negative: TRUE or FALSE * Failure: Negative * * Programmer: Quincey Koziol @@ -152,12 +284,11 @@ static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info, */ htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, - const H5S_t *mem_space, const H5D_type_info_t *type_info, - const H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist) + const H5S_t *mem_space, const H5D_type_info_t *type_info, H5P_genplist_t *dx_plist) { int local_cause = 0; /* Local reason(s) for breaking collective mode */ int global_cause = 0; /* Global reason(s) for breaking collective mode */ - htri_t ret_value; /* Return value */ + htri_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -206,11 +337,6 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, * use collective IO will defer until each chunk IO is reached. */ - /* Don't allow collective operations if filters need to be applied */ - if(io_info->dset->shared->layout.type == H5D_CHUNKED && - io_info->dset->shared->dcpl_cache.pline.nused > 0) - local_cause |= H5D_MPIO_FILTERS; - /* Check for independent I/O */ if(local_cause & H5D_MPIO_SET_INDEPENDENT) global_cause = local_cause; @@ -300,6 +426,113 @@ done: /*------------------------------------------------------------------------- + * Function: H5D__mpio_array_gatherv + * + * Purpose: Given an array, specified in local_array, by each processor + * calling this function, gathers each array into a single + * array which is then either gathered to the processor + * specified by root, when allgather is false, or is + * distributed back to all processors when allgather is true. + * + * The size of each entry and number of entries in the array + * contributed by an individual processor should be specified + * in array_entry_size and local_array_num_entries, + * respectively. + * + * The number of processors participating in the gather + * operation should be specified for nprocs. + * + * The MPI communicator to use should be specified for comm. + * + * If the sort_func argument is supplied, the array is sorted + * before the function returns. + * + * Note: if allgather is specified as true, root is ignored. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jordan Henderson + * Sunday, April 9th, 2017 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries, + size_t array_entry_size, void **_gathered_array, size_t *_gathered_array_num_entries, + int nprocs, hbool_t allgather, int root, MPI_Comm comm, int (*sort_func)(const void *, const void *)) +{ + size_t gathered_array_num_entries = 0; /* The size of the newly-constructed array */ + size_t i; + void *gathered_array = NULL; /* The newly-constructed array returned to the caller */ + int *receive_counts_array = NULL; /* Array containing number of entries each process is contributing */ + int *displacements_array = NULL; /* Array of displacements where each process places its data in the final array */ + int mpi_code; + int sendcount; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(_gathered_array); + HDassert(_gathered_array_num_entries); + + /* Determine the size of the end result array */ + if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_array_num_entries, &gathered_array_num_entries, 1, MPI_INT, MPI_SUM, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + + /* If 0 entries resulted from the collective operation, no one is writing anything */ + if (gathered_array_num_entries > 0) { + if (NULL == (gathered_array = H5MM_malloc(gathered_array_num_entries * array_entry_size))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate gathered array") + + if (NULL == (receive_counts_array = (int *) H5MM_malloc((size_t) nprocs * sizeof(int)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive counts array") + + if (NULL == (displacements_array = (int *) H5MM_malloc((size_t) nprocs * sizeof(int)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive displacements array") + + /* Inform each process of how many entries each other process is contributing to the resulting array */ + if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&local_array_num_entries, 1, MPI_INT, receive_counts_array, 1, MPI_INT, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + + /* Multiply each receive count by the size of the array entry, since the data is sent as bytes */ + for (i = 0; i < (size_t) nprocs; i++) + H5_CHECKED_ASSIGN(receive_counts_array[i], int, (size_t) receive_counts_array[i] * array_entry_size, size_t); + + /* Set receive buffer offsets for MPI_Allgatherv */ + displacements_array[0] = 0; + for (i = 1; i < (size_t) nprocs; i++) + displacements_array[i] = displacements_array[i - 1] + receive_counts_array[i - 1]; + + H5_CHECKED_ASSIGN(sendcount, int, local_array_num_entries * array_entry_size, size_t); + + if (allgather) { + if (MPI_SUCCESS != (mpi_code = MPI_Allgatherv(local_array, sendcount, MPI_BYTE, + gathered_array, receive_counts_array, displacements_array, MPI_BYTE, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code) + } /* end if */ + else { + if (MPI_SUCCESS != (mpi_code = MPI_Gatherv(local_array, sendcount, MPI_BYTE, + gathered_array, receive_counts_array, displacements_array, MPI_BYTE, root, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code) + } /* end else */ + + if (sort_func) HDqsort(gathered_array, gathered_array_num_entries, array_entry_size, sort_func); + } /* end if */ + + *_gathered_array = gathered_array; + *_gathered_array_num_entries = gathered_array_num_entries; + +done: + if (receive_counts_array) + H5MM_free(receive_counts_array); + if (displacements_array) + H5MM_free(displacements_array); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_array_gatherv() */ + + +/*------------------------------------------------------------------------- * Function: H5D__ioinfo_xfer_mode * * Purpose: Switch to between collective & independent MPI I/O @@ -398,7 +631,7 @@ H5D__mpio_get_min_chunk(const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, FUNC_ENTER_STATIC /* Get the number of chunks to perform I/O on */ - num_chunkf = H5SL_count(fm->sel_chunks); + H5_CHECKED_ASSIGN(num_chunkf, int, H5SL_count(fm->sel_chunks), size_t) /* Determine the minimum # of chunks for all processes */ if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&num_chunkf, min_chunkf, 1, MPI_INT, MPI_MIN, io_info->comm))) @@ -480,7 +713,7 @@ H5D__contig_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish shared collective MPI-IO") /* Obtain the data transfer properties */ - if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id))) + if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list") /* Set the actual I/O mode property. internal_collective_io will not break to @@ -527,7 +760,7 @@ H5D__contig_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't finish shared collective MPI-IO") /* Obtain the data transfer properties */ - if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id))) + if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list") /* Set the actual I/O mode property. internal_collective_io will not break to @@ -599,12 +832,13 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf HDassert(fm); /* Obtain the data transfer properties */ - if(NULL == (dx_plist = H5I_object(io_info->raw_dxpl_id))) + if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(io_info->raw_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - /* Check the optional property list on what to do with collective chunk IO. */ + /* Check the optional property list for the collective chunk IO optimization option */ if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_HARD_NAME, &chunk_opt_mode) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option") + if(H5FD_MPIO_CHUNK_ONE_IO == chunk_opt_mode) io_option = H5D_ONE_LINK_CHUNK_IO; /*no opt*/ /* direct request to multi-chunk-io */ @@ -620,13 +854,13 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf if((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") - /* Get the chunk optimization option */ + /* Get the chunk optimization option threshold */ if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_NUM_NAME, &one_link_chunk_io_threshold) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option") + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get chunk optimization option threshold value") /* step 1: choose an IO option */ /* If the average number of chunk per process is greater than a threshold, we will do one link chunked IO. */ - if((unsigned)sum_chunk / mpi_size >= one_link_chunk_io_threshold) + if((unsigned)sum_chunk / (unsigned)mpi_size >= one_link_chunk_io_threshold) io_option = H5D_ONE_LINK_CHUNK_IO_MORE_OPT; #ifdef H5_HAVE_INSTRUMENTED_LIBRARY else @@ -681,19 +915,46 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf #endif /* step 2: Go ahead to do IO.*/ - if(H5D_ONE_LINK_CHUNK_IO == io_option || H5D_ONE_LINK_CHUNK_IO_MORE_OPT == io_option) { - if(H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, dx_plist) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO") - } /* end if */ - /* direct request to multi-chunk-io */ - else if(H5D_MULTI_CHUNK_IO == io_option) { - if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO") - } /* end if */ - else { /* multiple chunk IO via threshold */ - if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO") - } /* end else */ + switch (io_option) { + case H5D_ONE_LINK_CHUNK_IO: + case H5D_ONE_LINK_CHUNK_IO_MORE_OPT: + /* Check if there are any filters in the pipeline */ + if(io_info->dset->shared->dcpl_cache.pline.nused > 0) { + /* For now, Multi-chunk IO must be forced for parallel filtered read, + * so that data can be unfiltered as it is received. There is significant + * complexity in unfiltering the data when it is read all at once into a + * single buffer. + */ + if (io_info->op_type == H5D_IO_OP_READ) { + if(H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO") + } /* end if */ + else { + if(H5D__link_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish filtered linked chunk MPI-IO") + } /* end else */ + } /* end if */ + else { + /* Perform unfiltered link chunk collective IO */ + if(H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, dx_plist) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO") + } /* end else */ + break; + + case H5D_MULTI_CHUNK_IO: /* direct request to do multi-chunk IO */ + default: /* multiple chunk IO via threshold */ + /* Check if there are any filters in the pipeline */ + if(io_info->dset->shared->dcpl_cache.pline.nused > 0) { + if(H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, dx_plist) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO") + } /* end if */ + else { + /* Perform unfiltered multi chunk collective IO */ + if(H5D__multi_chunk_collective_io(io_info, type_info, fm, dx_plist) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO") + } /* end else */ + break; + } /* end switch */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -1093,6 +1354,232 @@ if(H5DEBUG(D)) /*------------------------------------------------------------------------- + * Function: H5D__link_chunk_filtered_collective_io + * + * Purpose: Routine for one collective IO with one MPI derived datatype + * to link with all filtered chunks + * + * 1. Construct a list of selected chunks in the collective IO + * operation + * A. If any chunk is being written to by more than 1 + * process, the process writing to the chunk which + * currently has the least amount of chunks assigned + * to it becomes the new owner (in the case of ties, + * the lowest MPI rank becomes the new owner) + * 2. If the operation is a write operation + * A. Loop through each chunk in the operation + * I. If this is not a full overwrite of the chunk + * a) Read the chunk from file and pass the chunk + * through the filter pipeline in reverse order + * (Unfilter the chunk) + * II. Update the chunk data with the modifications from + * the owning process + * III. Receive any modification data from other + * processes and update the chunk data with these + * modifications + * IV. Filter the chunk + * B. Contribute the modified chunks to an array gathered + * by all processes which contains the new sizes of + * every chunk modified in the collective IO operation + * C. All processes collectively re-allocate each chunk + * from the gathered array with their new sizes after + * the filter operation + * D. If this process has any chunks selected in the IO + * operation, create an MPI derived type for memory and + * file to write out the process' selected chunks to the + * file + * E. Perform the collective write + * F. All processes collectively re-insert each modified + * chunk from the gathered array into the chunk index + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jordan Henderson + * Friday, Nov. 4th, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist) +{ + H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ + H5D_filtered_collective_io_info_t *collective_chunk_list = NULL; /* The list of chunks used during collective operations */ + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode = H5D_MPIO_LINK_CHUNK; /* The actual chunk IO optimization mode */ + H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CHUNK_COLLECTIVE; /* The chunk IO mode used (Independent vs Collective) */ + H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ + MPI_Datatype mem_type = MPI_BYTE; + MPI_Datatype file_type = MPI_BYTE; + hbool_t mem_type_is_derived = FALSE; + hbool_t file_type_is_derived = FALSE; + size_t chunk_list_num_entries; + size_t collective_chunk_list_num_entries; + size_t *num_chunks_selected_array = NULL; /* Array of number of chunks selected on each process */ + size_t i; /* Local index variable */ + int mpi_rank, mpi_size, mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(io_info); + HDassert(type_info); + HDassert(fm); + HDassert(dx_plist); + + /* Obtain the current rank of the process and the number of processes */ + if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") + if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") + + /* Set the actual-chunk-opt-mode property. */ + if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, &actual_chunk_opt_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk opt mode property") + + /* Set the actual-io-mode property. + * Link chunk filtered I/O does not break to independent, so can set right away + */ + if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_IO_MODE_NAME, &actual_io_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual io mode property") + + /* Build a list of selected chunks in the collective io operation */ + if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list") + + if (io_info->op_type == H5D_IO_OP_WRITE) { /* Filtered collective write */ + H5D_chk_idx_info_t index_info; + H5D_chunk_ud_t udata; + hsize_t mpi_buf_count; + + /* Construct chunked index info */ + index_info.f = io_info->dset->oloc.file; + index_info.dxpl_id = io_info->md_dxpl_id; + index_info.pline = &(io_info->dset->shared->dcpl_cache.pline); + index_info.layout = &(io_info->dset->shared->layout.u.chunk); + index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk); + + /* Set up chunk information for insertion to chunk index */ + udata.common.layout = index_info.layout; + udata.common.storage = index_info.storage; + udata.filter_mask = 0; + + /* Iterate through all the chunks in the collective write operation, + * updating each chunk with the data modifications from other processes, + * then re-filtering the chunk. + */ + for (i = 0; i < chunk_list_num_entries; i++) + if (mpi_rank == chunk_list[i].owners.new_owner) + if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry") + + /* Gather the new chunk sizes to all processes for a collective reallocation + * of the chunks in the file. + */ + if (H5D__mpio_array_gatherv(chunk_list, chunk_list_num_entries, sizeof(H5D_filtered_collective_io_info_t), + (void **) &collective_chunk_list, &collective_chunk_list_num_entries, mpi_size, + true, 0, io_info->comm, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes") + + /* Collectively re-allocate the modified chunks (from each process) in the file */ + for (i = 0; i < collective_chunk_list_num_entries; i++) { + hbool_t insert; + + if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[i].chunk_states.chunk_current, + &collective_chunk_list[i].chunk_states.new_chunk, &insert, collective_chunk_list[i].scaled) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") + } /* end for */ + + if (NULL == (num_chunks_selected_array = (size_t *) H5MM_malloc((size_t) mpi_size * sizeof(size_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array") + + if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&chunk_list_num_entries, 1, MPI_UNSIGNED_LONG_LONG, num_chunks_selected_array, + 1, MPI_UNSIGNED_LONG_LONG, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + + /* If this process has any chunks selected, create a MPI type for collectively + * writing out the chunks to file. Otherwise, the process contributes to the + * collective write with a none type. + */ + if (chunk_list_num_entries) { + size_t offset; + + /* During the collective re-allocation of chunks in the file, the record for each + * chunk is only updated in the collective array, not in the local copy of chunks on each + * process. However, each process needs the updated chunk records so that they can create + * a MPI type for the collective write that will write to the chunk's possible new locations + * in the file instead of the old ones. This ugly hack seems to be the best solution to + * copy the information back to the local array and avoid having to modify the collective + * write type function in an ugly way so that it will accept the collective array instead + * of the local array. This works correctly because the array gather function guarantees + * that the chunk data in the collective array is ordered in blocks by rank. + */ + for (i = 0, offset = 0; i < (size_t) mpi_rank; i++) + offset += num_chunks_selected_array[i]; + + HDmemcpy(chunk_list, &collective_chunk_list[offset], num_chunks_selected_array[mpi_rank] * sizeof(H5D_filtered_collective_io_info_t)); + + /* Create single MPI type encompassing each selection in the dataspace */ + if (H5D__mpio_filtered_collective_write_type(chunk_list, chunk_list_num_entries, + &mem_type, &mem_type_is_derived, &file_type, &file_type_is_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "couldn't create MPI link chunk I/O type") + + /* Override the write buffer to point to the address of the first + * chunk data buffer + */ + io_info->u.wbuf = chunk_list[0].buf; + } /* end if */ + + /* We have a single, complicated MPI datatype for both memory & file */ + mpi_buf_count = (mem_type_is_derived && file_type_is_derived) ? (hsize_t) 1 : (hsize_t) 0; + + /* Set up the base storage address for this operation */ + ctg_store.contig.dset_addr = 0; /* Write address must be set to address 0 */ + io_info->store = &ctg_store; + + /* Perform I/O */ + if (H5D__final_collective_io(io_info, type_info, mpi_buf_count, &file_type, &mem_type) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO") + + /* Participate in the collective re-insertion of all chunks modified + * in this iteration into the chunk index + */ + for (i = 0; i < collective_chunk_list_num_entries; i++) { + udata.chunk_block = collective_chunk_list[i].chunk_states.new_chunk; + udata.common.scaled = collective_chunk_list[i].scaled; + udata.chunk_idx = collective_chunk_list[i].index; + + if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index") + } /* end for */ + } /* end if */ + +done: + /* Free resources used by a process which had some selection */ + if (chunk_list) { + for (i = 0; i < chunk_list_num_entries; i++) + if (chunk_list[i].buf) + H5MM_free(chunk_list[i].buf); + + H5MM_free(chunk_list); + } /* end if */ + + if (num_chunks_selected_array) + H5MM_free(num_chunks_selected_array); + if (collective_chunk_list) + H5MM_free(collective_chunk_list); + + /* Free the MPI buf and file types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__link_chunk_filtered_collective_io() */ + + +/*------------------------------------------------------------------------- * Function: H5D__multi_chunk_collective_io * * Purpose: To do IO per chunk according to IO mode(collective/independent/none) @@ -1225,7 +1712,7 @@ if(H5DEBUG(D)) * to ease switching between to mixed I/O without checking the current * value of the property. You can see the definition in H5Ppublic.h */ - actual_io_mode = actual_io_mode | H5D_MPIO_CHUNK_COLLECTIVE; + actual_io_mode = (H5D_mpio_actual_io_mode_t) (actual_io_mode | H5D_MPIO_CHUNK_COLLECTIVE); } /* end if */ else { @@ -1265,7 +1752,7 @@ if(H5DEBUG(D)) mspace = chunk_info->mspace; /* Update the local variable tracking the dxpl's actual io mode. */ - actual_io_mode = actual_io_mode | H5D_MPIO_CHUNK_INDEPENDENT; + actual_io_mode = (H5D_mpio_actual_io_mode_t) (actual_io_mode | H5D_MPIO_CHUNK_INDEPENDENT); } /* end if */ else { fspace = mspace = NULL; @@ -1306,6 +1793,314 @@ done: /*------------------------------------------------------------------------- + * Function: H5D__multi_chunk_filtered_collective_io + * + * Purpose: To do filtered collective IO iteratively to save on memory. + * While link_chunk_filtered_collective_io will construct and + * work on a list of all of the chunks selected in the IO + * operation at once, this function works iteratively on a set + * of chunks at a time; at most one chunk per rank per + * iteration. + * + * 1. Construct a list of selected chunks in the collective IO + * operation + * A. If any chunk is being written to by more than 1 + * process, the process writing to the chunk which + * currently has the least amount of chunks assigned + * to it becomes the new owner (in the case of ties, + * the lowest MPI rank becomes the new owner) + * 2. If the operation is a read operation + * A. Loop through each chunk in the operation + * I. Read the chunk from the file + * II. Unfilter the chunk + * III. Scatter the read chunk data to the user's buffer + * 3. If the operation is a write operation + * A. Loop through each chunk in the operation + * I. If this is not a full overwrite of the chunk + * a) Read the chunk from file and pass the chunk + * through the filter pipeline in reverse order + * (Unfilter the chunk) + * II. Update the chunk data with the modifications from + * the owning process + * III. Receive any modification data from other + * processes and update the chunk data with these + * modifications + * IV. Filter the chunk + * V. Contribute the chunk to an array gathered by + * all processes which contains every chunk + * modified in this iteration (up to one chunk + * per process, some processes may not have a + * selection/may have less chunks to work on than + * other processes) + * VI. All processes collectively re-allocate each + * chunk from the gathered array with their new + * sizes after the filter operation + * VII. Proceed with the collective write operation + * for the chunks modified on this iteration + * VIII. All processes collectively re-insert each + * chunk from the gathered array into the chunk + * index + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jordan Henderson + * Friday, Dec. 2nd, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist) +{ + H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ + H5D_filtered_collective_io_info_t *collective_chunk_list = NULL; /* The list of chunks used during collective operations */ + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode = H5D_MPIO_MULTI_CHUNK; /* The actual chunk IO optimization mode */ + H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CHUNK_COLLECTIVE; /* The chunk IO mode used (Independent vs Collective) */ + H5D_storage_t store; /* union of EFL and chunk pointer in file space */ + H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ + H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ + MPI_Datatype *file_type_array = NULL; + MPI_Datatype *mem_type_array = NULL; + hbool_t *file_type_is_derived_array = NULL; + hbool_t *mem_type_is_derived_array = NULL; + hbool_t *has_chunk_selected_array = NULL; /* Array of whether or not each process is contributing a chunk to each iteration */ + size_t chunk_list_num_entries; + size_t collective_chunk_list_num_entries; + size_t i, j; /* Local index variable */ + int mpi_rank, mpi_size, mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(io_info); + HDassert(type_info); + HDassert(fm); + HDassert(dx_plist); + + /* Obtain the current rank of the process and the number of processes */ + if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") + if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") + + /* Set the actual chunk opt mode property */ + if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_CHUNK_OPT_MODE_NAME, &actual_chunk_opt_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk opt mode property") + + /* Set the actual_io_mode property. + * Multi chunk I/O does not break to independent, so can set right away + */ + if (H5P_set(dx_plist, H5D_MPIO_ACTUAL_IO_MODE_NAME, &actual_io_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "couldn't set actual chunk io mode property") + + /* Build a list of selected chunks in the collective IO operation */ + if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list") + + /* Set up contiguous I/O info object */ + HDmemcpy(&ctg_io_info, io_info, sizeof(ctg_io_info)); + ctg_io_info.store = &ctg_store; + ctg_io_info.layout_ops = *H5D_LOPS_CONTIG; + + /* Initialize temporary contiguous storage info */ + ctg_store.contig.dset_size = (hsize_t) io_info->dset->shared->layout.u.chunk.size; + ctg_store.contig.dset_addr = 0; + + /* Set dataset storage for I/O info */ + io_info->store = &store; + + if (io_info->op_type == H5D_IO_OP_READ) { /* Filtered collective read */ + for (i = 0; i < chunk_list_num_entries; i++) + if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't process chunk entry") + } /* end if */ + else { /* Filtered collective write */ + H5D_chk_idx_info_t index_info; + H5D_chunk_ud_t udata; + size_t max_num_chunks; + hsize_t mpi_buf_count; + + /* Construct chunked index info */ + index_info.f = io_info->dset->oloc.file; + index_info.dxpl_id = io_info->md_dxpl_id; + index_info.pline = &(io_info->dset->shared->dcpl_cache.pline); + index_info.layout = &(io_info->dset->shared->layout.u.chunk); + index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk); + + /* Set up chunk information for insertion to chunk index */ + udata.common.layout = index_info.layout; + udata.common.storage = index_info.storage; + udata.filter_mask = 0; + + /* Retrieve the maximum number of chunks being written among all processes */ + if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&chunk_list_num_entries, &max_num_chunks, + 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + + /* If no one is writing anything at all, end the operation */ + if (!(max_num_chunks > 0)) HGOTO_DONE(SUCCEED); + + /* Allocate arrays for storing MPI file and mem types and whether or not the + * types were derived. + */ + if (NULL == (file_type_array = (MPI_Datatype *) H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type array") + + if (NULL == (file_type_is_derived_array = (hbool_t *) H5MM_calloc(max_num_chunks * sizeof(hbool_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type is derived array") + + if (NULL == (mem_type_array = (MPI_Datatype *) H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type array") + + if (NULL == (mem_type_is_derived_array = (hbool_t *) H5MM_calloc(max_num_chunks * sizeof(hbool_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type is derived array") + + /* Iterate over the max number of chunks among all processes, as this process could + * have no chunks left to work on, but it still needs to participate in the collective + * re-allocation and re-insertion of chunks modified by other processes. + */ + for (i = 0; i < max_num_chunks; i++) { + /* Check if this process has a chunk to work on for this iteration */ + hbool_t have_chunk_to_process = (i < chunk_list_num_entries) && (mpi_rank == chunk_list[i].owners.new_owner); + + if (have_chunk_to_process) + if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry") + + /* Gather the new chunk sizes to all processes for a collective re-allocation + * of the chunks in the file + */ + if (H5D__mpio_array_gatherv(&chunk_list[i], have_chunk_to_process ? 1 : 0, sizeof(H5D_filtered_collective_io_info_t), + (void **) &collective_chunk_list, &collective_chunk_list_num_entries, mpi_size, + true, 0, io_info->comm, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes") + + /* Participate in the collective re-allocation of all chunks modified + * in this iteration. + */ + for (j = 0; j < collective_chunk_list_num_entries; j++) { + hbool_t insert = FALSE; + + if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[j].chunk_states.chunk_current, + &collective_chunk_list[j].chunk_states.new_chunk, &insert, chunk_list[j].scaled) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") + } /* end for */ + + if (NULL == (has_chunk_selected_array = (hbool_t *) H5MM_malloc((size_t) mpi_size * sizeof(hbool_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array") + + if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&have_chunk_to_process, 1, MPI_C_BOOL, has_chunk_selected_array, + 1, MPI_C_BOOL, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + + /* If this process has a chunk to work on, create a MPI type for the + * memory and file for writing out the chunk + */ + if (have_chunk_to_process) { + size_t offset; + int mpi_type_count; + + for (j = 0, offset = 0; j < (size_t) mpi_rank; j++) + offset += has_chunk_selected_array[j]; + + /* Collect the new chunk info back to the local copy, since only the record in the + * collective array gets updated by the chunk re-allocation */ + HDmemcpy(&chunk_list[i].chunk_states.new_chunk, &collective_chunk_list[offset].chunk_states.new_chunk, sizeof(chunk_list[i].chunk_states.new_chunk)); + + H5_CHECKED_ASSIGN(mpi_type_count, int, chunk_list[i].chunk_states.new_chunk.length, hsize_t); + + /* Create MPI memory type for writing to chunk */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &mem_type_array[i]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type_array[i]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + mem_type_is_derived_array[i] = TRUE; + + /* Create MPI file type for writing to chunk */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &file_type_array[i]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type_array[i]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + file_type_is_derived_array[i] = TRUE; + + mpi_buf_count = 1; + + /* Set up the base storage address for this operation */ + ctg_store.contig.dset_addr = chunk_list[i].chunk_states.new_chunk.offset; + + /* Override the write buffer to point to the address of the + * chunk data buffer + */ + ctg_io_info.u.wbuf = chunk_list[i].buf; + } /* end if */ + else { + mem_type_array[i] = file_type_array[i] = MPI_BYTE; + mpi_buf_count = 0; + } /* end else */ + + /* Perform the I/O */ + if (H5D__final_collective_io(&ctg_io_info, type_info, mpi_buf_count, &file_type_array[i], &mem_type_array[i]) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO") + + /* Participate in the collective re-insertion of all chunks modified + * in this iteration into the chunk index + */ + for (j = 0; j < collective_chunk_list_num_entries; j++) { + udata.chunk_block = collective_chunk_list[j].chunk_states.new_chunk; + udata.common.scaled = collective_chunk_list[j].scaled; + udata.chunk_idx = collective_chunk_list[j].index; + + if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index") + } /* end for */ + + if (collective_chunk_list){ + H5MM_free(collective_chunk_list); + collective_chunk_list = NULL; + } /* end if */ + if (has_chunk_selected_array){ + H5MM_free(has_chunk_selected_array); + has_chunk_selected_array = NULL; + } /* end if */ + } /* end for */ + + /* Free the MPI file and memory types, if they were derived */ + for (i = 0; i < max_num_chunks; i++) { + if (file_type_is_derived_array[i]) + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type_array[i]))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + if (mem_type_is_derived_array[i]) + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type_array[i]))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } /* end for */ + } /* end else */ + +done: + if (chunk_list) { + for (i = 0; i < chunk_list_num_entries; i++) + if (chunk_list[i].buf) + H5MM_free(chunk_list[i].buf); + + H5MM_free(chunk_list); + } /* end if */ + + if (collective_chunk_list) + H5MM_free(collective_chunk_list); + if (file_type_array) + H5MM_free(file_type_array); + if (mem_type_array) + H5MM_free(mem_type_array); + if (file_type_is_derived_array) + H5MM_free(file_type_is_derived_array); + if (mem_type_is_derived_array) + H5MM_free(mem_type_is_derived_array); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__multi_chunk_filtered_collective_io() */ + + +/*------------------------------------------------------------------------- * Function: H5D__inter_collective_io * * Purpose: Routine for the shared part of collective IO between multiple chunk @@ -1472,7 +2267,7 @@ if(H5DEBUG(D)) static int H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2) { - haddr_t addr1, addr2; + haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF; FUNC_ENTER_STATIC_NOERR @@ -1484,6 +2279,67 @@ H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2) /*------------------------------------------------------------------------- + * Function: H5D__cmp_filtered_collective_io_info_entry + * + * Purpose: Routine to compare filtered collective chunk io info + * entries + * + * Description: Callback for qsort() to compare filtered collective chunk + * io info entries + * + * Return: -1, 0, 1 + * + * Programmer: Jordan Henderson + * Wednesday, Nov. 30th, 2016 + * + *------------------------------------------------------------------------- + */ +static int +H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2) +{ + haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF; + + FUNC_ENTER_STATIC_NOERR + + addr1 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry1)->chunk_states.new_chunk.offset; + addr2 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry2)->chunk_states.new_chunk.offset; + + FUNC_LEAVE_NOAPI(H5F_addr_cmp(addr1, addr2)) +} /* end H5D__cmp_filtered_collective_io_info_entry() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__cmp_filtered_collective_io_info_entry_owner + * + * Purpose: Routine to compare filtered collective chunk io info + * entries's original owner fields + * + * Description: Callback for qsort() to compare filtered collective chunk + * io info entries's original owner fields + * + * Return: The difference between the two + * H5D_filtered_collective_io_info_t's original owner fields + * + * Programmer: Jordan Henderson + * Monday, Apr. 10th, 2017 + * + *------------------------------------------------------------------------- + */ +static int +H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2) +{ + int owner1 = -1, owner2 = -1; + + FUNC_ENTER_STATIC_NOERR + + owner1 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry1)->owners.original_owner; + owner2 = ((const H5D_filtered_collective_io_info_t *) filtered_collective_io_info_entry2)->owners.original_owner; + + FUNC_LEAVE_NOAPI(owner1 - owner2) +} /* end H5D__cmp_filtered_collective_io_info_entry_owner() */ + + +/*------------------------------------------------------------------------- * Function: H5D__sort_chunk * * Purpose: Routine to sort chunks in increasing order of chunk address @@ -1557,7 +2413,7 @@ if(H5DEBUG(D)) HDfprintf(H5DEBUG(D), "Coming inside H5D_OBTAIN_ALL_CHUNK_ADDR_COL\n"); #endif /* Allocate array for chunk addresses */ - if(NULL == (total_chunk_addr_array = H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks))) + if(NULL == (total_chunk_addr_array = (haddr_t *)H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate memory chunk address array") /* Retrieve all the chunk addresses with process 0 */ @@ -1581,7 +2437,7 @@ if(H5DEBUG(D)) /* Iterate over all chunks for this process */ while(chunk_node) { - if(NULL == (chunk_info = H5SL_item(chunk_node))) + if(NULL == (chunk_info = (H5D_chunk_info_t *)H5SL_item(chunk_node))) HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL,"couldn't get chunk info from skipped list") if(many_chunk_opt == H5D_OBTAIN_ONE_CHUNK_ADDR_IND) { @@ -1666,7 +2522,7 @@ static herr_t H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm, H5P_genplist_t *dx_plist, uint8_t assign_io_mode[], haddr_t chunk_addr[]) { - int total_chunks; + size_t total_chunks; unsigned percent_nproc_per_chunk, threshold_nproc_per_chunk; uint8_t* io_mode_info = NULL; uint8_t* recv_io_mode_info = NULL; @@ -1676,7 +2532,8 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm, H5D_chunk_info_t* chunk_info; int mpi_size, mpi_rank; MPI_Comm comm; - int ic, root; + int root; + size_t ic; int mpi_code; #ifdef H5_HAVE_INSTRUMENTED_LIBRARY int new_value; @@ -1697,7 +2554,7 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm, HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") /* Setup parameters */ - H5_CHECKED_ASSIGN(total_chunks, int, fm->layout->u.chunk.nchunks, hsize_t); + H5_CHECKED_ASSIGN(total_chunks, size_t, fm->layout->u.chunk.nchunks, hsize_t); if(H5P_get(dx_plist, H5D_XFER_MPIO_CHUNK_OPT_RATIO_NAME, &percent_nproc_per_chunk) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't get percent nproc per chunk") /* if ratio is 0, perform collective io */ @@ -1709,39 +2566,42 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm, HGOTO_DONE(SUCCEED) } /* end if */ - threshold_nproc_per_chunk = mpi_size * percent_nproc_per_chunk/100; + + threshold_nproc_per_chunk = (unsigned)mpi_size * percent_nproc_per_chunk/100; /* Allocate memory */ if(NULL == (io_mode_info = (uint8_t *)H5MM_calloc(total_chunks))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate I/O mode info buffer") - if(NULL == (mergebuf = H5MM_malloc((sizeof(haddr_t) + 1) * total_chunks))) + if(NULL == (mergebuf = (uint8_t *)H5MM_malloc((sizeof(haddr_t) + 1) * total_chunks))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mergebuf buffer") tempbuf = mergebuf + total_chunks; if(mpi_rank == root) - if(NULL == (recv_io_mode_info = (uint8_t *)H5MM_malloc(total_chunks * mpi_size))) + if(NULL == (recv_io_mode_info = (uint8_t *)H5MM_malloc(total_chunks * (size_t)mpi_size))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate recv I/O mode info buffer") /* Obtain the regularity and selection information for all chunks in this process. */ chunk_node = H5SL_first(fm->sel_chunks); while(chunk_node) { - chunk_info = H5SL_item(chunk_node); + chunk_info = (H5D_chunk_info_t *)H5SL_item(chunk_node); - io_mode_info[chunk_info->index] = H5D_CHUNK_SELECT_REG; /* this chunk is selected and is "regular" */ + io_mode_info[chunk_info->index] = H5D_CHUNK_SELECT_REG; /* this chunk is selected and is "regular" */ chunk_node = H5SL_next(chunk_node); } /* end while */ /* Gather all the information */ - if(MPI_SUCCESS != (mpi_code = MPI_Gather(io_mode_info, total_chunks, MPI_BYTE, recv_io_mode_info, total_chunks, MPI_BYTE, root, comm))) + H5_CHECK_OVERFLOW(total_chunks, size_t, int) + if(MPI_SUCCESS != (mpi_code = MPI_Gather(io_mode_info, (int)total_chunks, MPI_BYTE, + recv_io_mode_info, (int)total_chunks, MPI_BYTE, root, comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Gather failed", mpi_code) /* Calculate the mode for IO(collective, independent or none) at root process */ if(mpi_rank == root) { - int nproc; - int* nproc_per_chunk; + size_t nproc; + unsigned* nproc_per_chunk; /* pre-computing: calculate number of processes and regularity of the selection occupied in each chunk */ - if(NULL == (nproc_per_chunk = (int*)H5MM_calloc(total_chunks * sizeof(int)))) + if(NULL == (nproc_per_chunk = (unsigned*)H5MM_calloc(total_chunks * sizeof(unsigned)))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate nproc_per_chunk buffer") /* calculating the chunk address */ @@ -1751,7 +2611,7 @@ H5D__obtain_mpio_mode(H5D_io_info_t* io_info, H5D_chunk_map_t *fm, } /* end if */ /* checking for number of process per chunk and regularity of the selection*/ - for(nproc = 0; nproc < mpi_size; nproc++) { + for(nproc = 0; nproc < (size_t)mpi_size; nproc++) { uint8_t *tmp_recv_io_mode_info = recv_io_mode_info + (nproc * total_chunks); /* Calculate the number of process per chunk and adding irregular selection option */ @@ -1835,5 +2695,712 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__obtain_mpio_mode() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__construct_filtered_io_info_list + * + * Purpose: Constructs a list of entries which contain the necessary + * information for inter-process communication when performing + * collective io on filtered chunks. This list is used by + * each process when performing I/O on locally selected chunks + * and also in operations that must be collectively done + * on every chunk, such as chunk re-allocation, insertion of + * chunks into the chunk index, etc. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jordan Henderson + * Tuesday, January 10th, 2017 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5D_chunk_map_t *fm, H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries) +{ + H5D_filtered_collective_io_info_t *local_info_array = NULL; /* The list of initially selected chunks for this process */ + size_t num_chunks_selected; + size_t i; + int mpi_rank; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(io_info); + HDassert(type_info); + HDassert(fm); + HDassert(chunk_list); + HDassert(num_entries); + HDassert(TRUE == H5P_isa_class(io_info->raw_dxpl_id, H5P_DATASET_XFER)); + + if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") + + /* Each process builds a local list of the chunks they have selected */ + if ((num_chunks_selected = H5SL_count(fm->sel_chunks))) { + H5D_chunk_info_t *chunk_info; + H5D_chunk_ud_t udata; + H5SL_node_t *chunk_node; + hssize_t select_npoints; + hssize_t chunk_npoints; + + if (NULL == (local_info_array = (H5D_filtered_collective_io_info_t *) H5MM_malloc(num_chunks_selected * sizeof(H5D_filtered_collective_io_info_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate local io info array buffer") + + chunk_node = H5SL_first(fm->sel_chunks); + for (i = 0; chunk_node; i++) { + chunk_info = (H5D_chunk_info_t *) H5SL_item(chunk_node); + + /* Obtain this chunk's address */ + if (H5D__chunk_lookup(io_info->dset, io_info->md_dxpl_id, chunk_info->scaled, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") + + local_info_array[i].index = chunk_info->index; + local_info_array[i].chunk_states.chunk_current = local_info_array[i].chunk_states.new_chunk = udata.chunk_block; + local_info_array[i].num_writers = 0; + local_info_array[i].owners.original_owner = local_info_array[i].owners.new_owner = mpi_rank; + local_info_array[i].buf = NULL; + + local_info_array[i].async_info.num_receive_requests = 0; + local_info_array[i].async_info.receive_buffer_array = NULL; + local_info_array[i].async_info.receive_requests_array = NULL; + + HDmemcpy(local_info_array[i].scaled, chunk_info->scaled, sizeof(chunk_info->scaled)); + + if ((select_npoints = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") + local_info_array[i].io_size = (size_t) select_npoints * type_info->src_type_size; + + /* Currently the full overwrite status of a chunk is only obtained on a per-process + * basis. This means that if the total selection in the chunk, as determined by the combination + * of selections of all of the processes interested in the chunk, covers the entire chunk, + * the performance optimization of not reading the chunk from the file is still valid, but + * is not applied in the current implementation. Something like an appropriately placed + * MPI_Allreduce or a running total of the number of chunk points selected during chunk + * redistribution should suffice for implementing this case - JTH. + */ + if ((chunk_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") + local_info_array[i].full_overwrite = + (local_info_array[i].io_size >= (hsize_t) chunk_npoints * type_info->dst_type_size) ? TRUE : FALSE; + + chunk_node = H5SL_next(chunk_node); + } /* end for */ + } /* end if */ + + /* Redistribute shared chunks to new owners as necessary */ + if (io_info->op_type == H5D_IO_OP_WRITE) + if (H5D__chunk_redistribute_shared_chunks(io_info, type_info, fm, local_info_array, &num_chunks_selected) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks") + + *chunk_list = local_info_array; + *num_entries = num_chunks_selected; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__construct_filtered_io_info_list() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__chunk_redistribute_shared_chunks + * + * Purpose: When performing a collective write on a Dataset with + * filters applied, this function is used to redistribute any + * chunks which are selected by more than one process, so as + * to preserve file integrity after the write by ensuring + * that any shared chunks are only modified by one process. + * + * The current implementation follows this 3-phase process: + * + * - Collect everyone's list of chunks into one large list, + * sort the list in increasing order of chunk offset in the + * file and hand the list off to rank 0 + * + * - Rank 0 scans the list looking for matching runs of chunk + * offset in the file (corresponding to a shared chunk which + * has been selected by more than one rank in the I/O + * operation) and for each shared chunk, it redistributes + * the chunk to the process writing to the chunk which + * currently has the least amount of chunks assigned to it + * by modifying the "new_owner" field in each of the list + * entries corresponding to that chunk + * + * - After the chunks have been redistributed, rank 0 re-sorts + * the list in order of previous owner so that each rank + * will get back exactly the array that they contributed to + * the redistribution operation, with the "new_owner" field + * of each chunk they are modifying having possibly been + * modified. Rank 0 then scatters each segment of the list + * back to its corresponding rank + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jordan Henderson + * Monday, May 1, 2017 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5D_chunk_map_t *fm, H5D_filtered_collective_io_info_t *local_chunk_array, size_t *local_chunk_array_num_entries) +{ + H5D_filtered_collective_io_info_t *shared_chunks_info_array = NULL; /* The list of all chunks selected in the operation by all processes */ + H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__gather_mem */ + unsigned char **mod_data = NULL; /* Array of chunk modification data buffers sent by a process to new chunk owners */ + MPI_Request *send_requests = NULL; /* Array of MPI_Isend chunk modification data send requests */ + MPI_Status *send_statuses = NULL; /* Array of MPI_Isend chunk modification send statuses */ + hbool_t mem_iter_init = FALSE; + size_t shared_chunks_info_array_num_entries = 0; + size_t num_send_requests = 0; + size_t *num_assigned_chunks_array = NULL; + size_t i, last_assigned_idx; + int *send_counts = NULL; + int *send_displacements = NULL; + int scatter_recvcount_int; + int mpi_rank, mpi_size, mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(io_info); + HDassert(type_info); + HDassert(fm); + HDassert(local_chunk_array_num_entries); + + if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") + if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") + + if (*local_chunk_array_num_entries) + if (NULL == (send_requests = (MPI_Request *) H5MM_malloc(*local_chunk_array_num_entries * sizeof(MPI_Request)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send requests buffer") + + if (NULL == (mem_iter = (H5S_sel_iter_t *) H5MM_malloc(sizeof(H5S_sel_iter_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + + /* Gather every rank's list of chunks to rank 0 to allow it to perform the redistribution operation. After this + * call, the gathered list will initially be sorted in increasing order of chunk offset in the file. + */ + if (H5D__mpio_array_gatherv(local_chunk_array, *local_chunk_array_num_entries, sizeof(H5D_filtered_collective_io_info_t), + (void **) &shared_chunks_info_array, &shared_chunks_info_array_num_entries, mpi_size, + false, 0, io_info->comm, H5D__cmp_filtered_collective_io_info_entry) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather array") + + /* Rank 0 redistributes any shared chunks to new owners as necessary */ + if (mpi_rank == 0) { + if (NULL == (send_counts = (int *) H5MM_calloc((size_t) mpi_size * sizeof(int)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send counts buffer") + + if (NULL == (send_displacements = (int *) H5MM_malloc((size_t) mpi_size * sizeof(int)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send displacements buffer") + + if (NULL == (num_assigned_chunks_array = (size_t *) H5MM_calloc((size_t) mpi_size * sizeof(size_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate number of assigned chunks array") + + for (i = 0; i < shared_chunks_info_array_num_entries;) { + H5D_filtered_collective_io_info_t chunk_entry; + haddr_t last_seen_addr = shared_chunks_info_array[i].chunk_states.chunk_current.offset; + size_t set_begin_index = i; + size_t num_writers = 0; + int new_chunk_owner = shared_chunks_info_array[i].owners.original_owner; + + /* Process each set of duplicate entries caused by another process writing to the same chunk */ + do { + chunk_entry = shared_chunks_info_array[i]; + + send_counts[chunk_entry.owners.original_owner] += (int) sizeof(chunk_entry); + + /* The new owner of the chunk is determined by the process + * writing to the chunk which currently has the least amount + * of chunks assigned to it + */ + if (num_assigned_chunks_array[chunk_entry.owners.original_owner] < num_assigned_chunks_array[new_chunk_owner]) + new_chunk_owner = chunk_entry.owners.original_owner; + + num_writers++; + } while (++i < shared_chunks_info_array_num_entries && shared_chunks_info_array[i].chunk_states.chunk_current.offset == last_seen_addr); + + /* Set all of the chunk entries' "new_owner" fields */ + for (; set_begin_index < i; set_begin_index++) { + shared_chunks_info_array[set_begin_index].owners.new_owner = new_chunk_owner; + shared_chunks_info_array[set_begin_index].num_writers = num_writers; + } /* end for */ + + num_assigned_chunks_array[new_chunk_owner]++; + } /* end for */ + + /* Sort the new list in order of previous owner so that each original owner of a chunk + * entry gets that entry back, with the possibly newly-modified "new_owner" field + */ + HDqsort(shared_chunks_info_array, shared_chunks_info_array_num_entries, + sizeof(H5D_filtered_collective_io_info_t), H5D__cmp_filtered_collective_io_info_entry_owner); + + send_displacements[0] = 0; + for (i = 1; i < (size_t) mpi_size; i++) + send_displacements[i] = send_displacements[i - 1] + send_counts[i - 1]; + } /* end if */ + + /* Scatter the segments of the list back to each process */ + H5_CHECKED_ASSIGN(scatter_recvcount_int, int, *local_chunk_array_num_entries * sizeof(H5D_filtered_collective_io_info_t), size_t); + if (MPI_SUCCESS != (mpi_code = MPI_Scatterv(shared_chunks_info_array, send_counts, send_displacements, + MPI_BYTE, local_chunk_array, scatter_recvcount_int, MPI_BYTE, 0, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "unable to scatter shared chunks info buffer", mpi_code) + + if (shared_chunks_info_array) { + H5MM_free(shared_chunks_info_array); + shared_chunks_info_array = NULL; + } /* end if */ + + /* Now that the chunks have been redistributed, each process must send its modification data + * to the new owners of any of the chunks it previously possessed. Accordingly, each process + * must also issue asynchronous receives for any messages it may receive for each of the + * chunks it is assigned, in order to avoid potential deadlocking issues. + */ + if (*local_chunk_array_num_entries) + if (NULL == (mod_data = (unsigned char **) H5MM_malloc(*local_chunk_array_num_entries * sizeof(unsigned char *)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data buffer array") + + for (i = 0, last_assigned_idx = 0; i < *local_chunk_array_num_entries; i++) { + H5D_filtered_collective_io_info_t *chunk_entry = &local_chunk_array[i]; + + if (mpi_rank != chunk_entry->owners.new_owner) { + H5D_chunk_info_t *chunk_info = NULL; + unsigned char *mod_data_p = NULL; + hssize_t iter_nelmts; + size_t mod_data_size; + + /* Look up the chunk and get its file and memory dataspaces */ + if (NULL == (chunk_info = (H5D_chunk_info_t *) H5SL_search(fm->sel_chunks, &chunk_entry->index))) + HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list") + + /* Determine size of serialized chunk file dataspace, plus the size of + * the data being written + */ + if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to get encoded dataspace size") + + if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") + + mod_data_size += (size_t) iter_nelmts * type_info->src_type_size; + + if (NULL == (mod_data[num_send_requests] = (unsigned char *) H5MM_malloc(mod_data_size))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk modification send buffer") + + /* Serialize the chunk's file dataspace into the buffer */ + mod_data_p = mod_data[num_send_requests]; + if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to encode dataspace") + + /* Intialize iterator for memory selection */ + if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") + mem_iter_init = TRUE; + + /* Collect the modification data into the buffer */ + if (!H5D__gather_mem(io_info->u.wbuf, chunk_info->mspace, mem_iter, + (size_t) iter_nelmts, io_info->dxpl_cache, mod_data_p)) + HGOTO_ERROR(H5E_IO, H5E_CANTGATHER, FAIL, "couldn't gather from write buffer") + + /* Send modification data to new owner */ + H5_CHECK_OVERFLOW(mod_data_size, size_t, int) + H5_CHECK_OVERFLOW(chunk_entry->index, hsize_t, int) + if (MPI_SUCCESS != (mpi_code = MPI_Isend(mod_data[num_send_requests], (int) mod_data_size, MPI_BYTE, + chunk_entry->owners.new_owner, (int) chunk_entry->index, io_info->comm, &send_requests[num_send_requests]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Isend failed", mpi_code) + + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release memory selection iterator") + mem_iter_init = FALSE; + + num_send_requests++; + } /* end if */ + else { + /* Allocate all necessary buffers for an asynchronous receive operation */ + if (chunk_entry->num_writers > 1) { + MPI_Message message; + MPI_Status status; + size_t j; + + chunk_entry->async_info.num_receive_requests = (int) chunk_entry->num_writers - 1; + if (NULL == (chunk_entry->async_info.receive_requests_array = (MPI_Request *) H5MM_malloc((size_t) chunk_entry->async_info.num_receive_requests * sizeof(MPI_Request)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async requests array") + + if (NULL == (chunk_entry->async_info.receive_buffer_array = (unsigned char **) H5MM_malloc((size_t) chunk_entry->async_info.num_receive_requests * sizeof(unsigned char *)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async receive buffers") + + for (j = 0; j < chunk_entry->num_writers - 1; j++) { + int count = 0; + + /* Probe for a particular message from any process, removing that message + * from the receive queue in the process and allocating that much memory + * for the asynchronous receive + */ + if (MPI_SUCCESS != (mpi_code = MPI_Mprobe(MPI_ANY_SOURCE, (int) chunk_entry->index, io_info->comm, &message, &status))) + HMPI_GOTO_ERROR(FAIL, "MPI_Mprobe failed", mpi_code) + + if (MPI_SUCCESS != (mpi_code = MPI_Get_count(&status, MPI_BYTE, &count))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_count failed", mpi_code) + + HDassert(count >= 0); + if (NULL == (chunk_entry->async_info.receive_buffer_array[j] = (unsigned char *) H5MM_malloc((size_t) count * sizeof(char *)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data receive buffer") + + if (MPI_SUCCESS != (mpi_code = MPI_Imrecv(chunk_entry->async_info.receive_buffer_array[j], count, MPI_BYTE, + &message, &chunk_entry->async_info.receive_requests_array[j]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Imrecv failed", mpi_code) + } /* end for */ + } /* end if */ + + local_chunk_array[last_assigned_idx++] = local_chunk_array[i]; + } /* end else */ + } /* end for */ + + *local_chunk_array_num_entries = last_assigned_idx; + + /* Wait for all async send requests to complete before returning */ + if (num_send_requests) { + if (NULL == (send_statuses = (MPI_Status *) H5MM_malloc(num_send_requests * sizeof(MPI_Status)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send statuses buffer") + + H5_CHECK_OVERFLOW(num_send_requests, size_t, int); + if (MPI_SUCCESS != (mpi_code = MPI_Waitall((int) num_send_requests, send_requests, send_statuses))) + HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) + } /* end if */ + +done: + /* Now that all async send requests have completed, free up the send + * buffers used in the async operations + */ + for (i = 0; i < num_send_requests; i++) { + if (mod_data[i]) + H5MM_free(mod_data[i]); + } /* end for */ + + if (send_requests) + H5MM_free(send_requests); + if (send_statuses) + H5MM_free(send_statuses); + if (send_counts) + H5MM_free(send_counts); + if (send_displacements) + H5MM_free(send_displacements); + if (mod_data) + H5MM_free(mod_data); + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + if (mem_iter) + H5MM_free(mem_iter); + if (num_assigned_chunks_array) + H5MM_free(num_assigned_chunks_array); + if (shared_chunks_info_array) + H5MM_free(shared_chunks_info_array); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__chunk_redistribute_shared_chunks() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_filtered_collective_write_type + * + * Purpose: Constructs a MPI derived datatype for both the memory and + * the file for a collective write of filtered chunks. The + * datatype contains the offsets in the file and the locations + * of the filtered chunk data buffers. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jordan Henderson + * Tuesday, November 22, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_filtered_collective_write_type(H5D_filtered_collective_io_info_t *chunk_list, + size_t num_entries, MPI_Datatype *new_mem_type, hbool_t *mem_type_derived, + MPI_Datatype *new_file_type, hbool_t *file_type_derived) +{ + MPI_Aint *write_buf_array = NULL; /* Relative displacements of filtered chunk data buffers */ + MPI_Aint *file_offset_array = NULL; /* Chunk offsets in the file */ + int *length_array = NULL; /* Filtered Chunk lengths */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list); + HDassert(new_mem_type); + HDassert(mem_type_derived); + HDassert(new_file_type); + HDassert(file_type_derived); + + if (num_entries > 0) { + size_t i; + int mpi_code; + void *base_buf; + + H5_CHECK_OVERFLOW(num_entries, size_t, int); + + /* Allocate arrays */ + if (NULL == (length_array = (int *) H5MM_malloc((size_t) num_entries * sizeof(int)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for filtered collective write length array") + if (NULL == (write_buf_array = (MPI_Aint *) H5MM_malloc((size_t) num_entries * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for filtered collective write buf length array") + if (NULL == (file_offset_array = (MPI_Aint *) H5MM_malloc((size_t) num_entries * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for collective write offset array") + + /* Ensure the list is sorted in ascending order of offset in the file */ + HDqsort(chunk_list, num_entries, sizeof(H5D_filtered_collective_io_info_t), H5D__cmp_filtered_collective_io_info_entry); + + base_buf = chunk_list[0].buf; + for (i = 0; i < num_entries; i++) { + /* Set up the offset in the file, the length of the chunk data, and the relative + * displacement of the chunk data write buffer + */ + file_offset_array[i] = (MPI_Aint) chunk_list[i].chunk_states.new_chunk.offset; + length_array[i] = (int) chunk_list[i].chunk_states.new_chunk.length; + write_buf_array[i] = (MPI_Aint) chunk_list[i].buf - (MPI_Aint) base_buf; + } /* end for */ + + /* Create memory MPI type */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int) num_entries, length_array, write_buf_array, MPI_BYTE, new_mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + *mem_type_derived = TRUE; + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + + /* Create file MPI type */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int) num_entries, length_array, file_offset_array, MPI_BYTE, new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + *file_type_derived = TRUE; + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } /* end if */ + +done: + if (write_buf_array) + H5MM_free(write_buf_array); + if (file_offset_array) + H5MM_free(file_offset_array); + if (length_array) + H5MM_free(length_array); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_filtered_collective_write_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__filtered_collective_chunk_entry_io + * + * Purpose: Given an entry for a filtered chunk, performs the necessary + * steps for updating the chunk data during a collective + * write, or for reading the chunk from file during a + * collective read. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Jordan Henderson + * Wednesday, January 18, 2017 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry, + const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm) +{ + H5D_chunk_info_t *chunk_info = NULL; + H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__scatter_mem/H5D__gather_mem */ + unsigned char *mod_data = NULL; /* Chunk modification data sent by a process to a chunk's owner */ + unsigned filter_mask = 0; + hssize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */ + hssize_t extent_npoints; + hsize_t true_chunk_size; + hbool_t mem_iter_init = FALSE; + size_t buf_size; + size_t i; + H5S_t *dataspace = NULL; /* Other process' dataspace for the chunk */ + void *tmp_gath_buf = NULL; /* Temporary gather buffer for owner of the chunk to gather into from + application write buffer before scattering out to the chunk data buffer */ + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_entry); + HDassert(io_info); + HDassert(type_info); + HDassert(fm); + + /* Look up the chunk and get its file and memory dataspaces */ + if (NULL == (chunk_info = (H5D_chunk_info_t *) H5SL_search(fm->sel_chunks, &chunk_entry->index))) + HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list") + + if ((extent_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") + true_chunk_size = (hsize_t) extent_npoints * type_info->src_type_size; + + /* If the size of the filtered chunk is larger than the number of points in the + * chunk file space extent times the datatype size, allocate enough space to hold the + * whole filtered chunk. Otherwise, allocate a buffer equal to the size of the + * chunk so that the unfiltering operation doesn't have to grow the buffer. + */ + buf_size = MAX(chunk_entry->chunk_states.chunk_current.length, true_chunk_size); + + if (NULL == (chunk_entry->buf = H5MM_malloc(buf_size))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer") + + /* If this is not a full chunk overwrite or this is a read operation, the chunk must be + * read from the file and unfiltered. + */ + if (!chunk_entry->full_overwrite || io_info->op_type == H5D_IO_OP_READ) { + chunk_entry->chunk_states.new_chunk.length = chunk_entry->chunk_states.chunk_current.length; + + /* Currently, these chunk reads are done independently and will likely + * cause issues with collective metadata reads enabled. In the future, + * this should be refactored to use collective chunk reads - JTH */ + if (H5F_block_read(io_info->dset->oloc.file, H5FD_MEM_DRAW, chunk_entry->chunk_states.chunk_current.offset, + chunk_entry->chunk_states.new_chunk.length, H5AC_rawdata_dxpl_id, chunk_entry->buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk") + + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, &filter_mask, + io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb, + (size_t *) &chunk_entry->chunk_states.new_chunk.length, &buf_size, &chunk_entry->buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying") + } /* end if */ + else { + chunk_entry->chunk_states.new_chunk.length = true_chunk_size; + } /* end else */ + + /* Initialize iterator for memory selection */ + if (NULL == (mem_iter = (H5S_sel_iter_t *) H5MM_malloc(sizeof(H5S_sel_iter_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + + if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") + mem_iter_init = TRUE; + + if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") + + /* If this is a read operation, scatter the read chunk data to the user's buffer. + * + * If this is a write operation, update the chunk data buffer with the modifications + * from the current process, then apply any modifications from other processes. Finally, + * filter the newly-updated chunk. + */ + switch (io_info->op_type) { + case H5D_IO_OP_READ: + if (H5D__scatter_mem(chunk_entry->buf, chunk_info->mspace, mem_iter, + (size_t) iter_nelmts, io_info->dxpl_cache, io_info->u.rbuf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to read buffer") + break; + + case H5D_IO_OP_WRITE: + if (NULL == (tmp_gath_buf = H5MM_malloc((hsize_t) iter_nelmts * type_info->src_type_size))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate temporary gather buffer") + + /* Gather modification data from the application write buffer into a temporary buffer */ + if (!H5D__gather_mem(io_info->u.wbuf, chunk_info->mspace, mem_iter, + (size_t) iter_nelmts, io_info->dxpl_cache, tmp_gath_buf)) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't gather from write buffer") + + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + mem_iter_init = FALSE; + + /* Initialize iterator for file selection */ + if (H5S_select_iter_init(mem_iter, chunk_info->fspace, type_info->dst_type_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize file selection information") + mem_iter_init = TRUE; + + if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") + + /* Scatter the owner's modification data into the chunk data buffer according to + * the file space. + */ + if (H5D__scatter_mem(tmp_gath_buf, chunk_info->fspace, mem_iter, + (size_t) iter_nelmts, io_info->dxpl_cache, chunk_entry->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to chunk data buffer") + + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + mem_iter_init = FALSE; + + if (MPI_SUCCESS != (mpi_code = MPI_Waitall(chunk_entry->async_info.num_receive_requests, + chunk_entry->async_info.receive_requests_array, MPI_STATUSES_IGNORE))) + HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) + + /* For each asynchronous receive call previously posted, receive the chunk modification + * buffer from another rank and update the chunk data + */ + for (i = 0; i < (size_t) chunk_entry->async_info.num_receive_requests; i++) { + const unsigned char *mod_data_p; + + /* Decode the process' chunk file dataspace */ + mod_data_p = chunk_entry->async_info.receive_buffer_array[i]; + if (NULL == (dataspace = H5S_decode(&mod_data_p))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, FAIL, "unable to decode dataspace") + + if (H5S_select_iter_init(mem_iter, dataspace, type_info->dst_type_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") + mem_iter_init = TRUE; + + if ((iter_nelmts = H5S_GET_SELECT_NPOINTS(dataspace)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") + + /* Update the chunk data with the received modification data */ + if (H5D__scatter_mem(mod_data_p, dataspace, mem_iter, (size_t) iter_nelmts, + io_info->dxpl_cache, chunk_entry->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't scatter to write buffer") + + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + mem_iter_init = FALSE; + if (dataspace) { + if (H5S_close(dataspace) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") + dataspace = NULL; + } + H5MM_free(chunk_entry->async_info.receive_buffer_array[i]); + } /* end for */ + + /* Filter the chunk */ + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, 0, &filter_mask, + io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb, + (size_t *) &chunk_entry->chunk_states.new_chunk.length, &buf_size, &chunk_entry->buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed") + +#if H5_SIZEOF_SIZE_T > 4 + /* Check for the chunk expanding too much to encode in a 32-bit value */ + if (chunk_entry->chunk_states.new_chunk.length > ((size_t) 0xffffffff)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length") +#endif + + break; + default: + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid I/O operation") + } /* end switch */ + +done: + if (chunk_entry->async_info.receive_buffer_array) + H5MM_free(chunk_entry->async_info.receive_buffer_array); + if (chunk_entry->async_info.receive_requests_array) + H5MM_free(chunk_entry->async_info.receive_requests_array); + if (mod_data) + H5MM_free(mod_data); + if (tmp_gath_buf) + H5MM_free(tmp_gath_buf); + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + if (mem_iter) + H5MM_free(mem_iter); + if (dataspace) + if (H5S_close(dataspace) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__filtered_collective_chunk_entry_io() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index a6857b9..097fab7 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -617,6 +617,9 @@ H5_DLL herr_t H5D__select_write(const H5D_io_info_t *io_info, H5_DLL herr_t H5D__scatter_mem(const void *_tscat_buf, const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache, void *_buf); +H5_DLL size_t H5D__gather_mem(const void *_buf, + const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, + const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/); H5_DLL herr_t H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); @@ -666,6 +669,8 @@ H5_DLL herr_t H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *scaled, H5D_chunk_ud_t *udata); H5_DLL herr_t H5D__chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes); H5_DLL herr_t H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_t old_dim[]); +H5_DLL herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk, + H5F_block_t *new_chunk, hbool_t *need_insert, hsize_t scaled[]); H5_DLL herr_t H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id, hsize_t old_dim[]); H5_DLL herr_t H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, @@ -768,8 +773,7 @@ H5_DLL herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info, * memory and the file */ H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_type_info_t *type_info, const H5D_chunk_map_t *fm, - H5P_genplist_t *dx_plist); + const H5D_type_info_t *type_info, H5P_genplist_t *dx_plist); #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dscatgath.c b/src/H5Dscatgath.c index 4625c7a..0ae69ee 100644 --- a/src/H5Dscatgath.c +++ b/src/H5Dscatgath.c @@ -47,9 +47,6 @@ static herr_t H5D__scatter_file(const H5D_io_info_t *io_info, static size_t H5D__gather_file(const H5D_io_info_t *io_info, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, void *buf); -static size_t H5D__gather_mem(const void *_buf, - const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, - const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/); static herr_t H5D__compound_opt_read(size_t nelmts, const H5S_t *mem_space, H5S_sel_iter_t *iter, const H5D_dxpl_cache_t *dxpl_cache, const H5D_type_info_t *type_info, void *user_buf/*out*/); @@ -303,6 +300,7 @@ H5D__scatter_mem (const void *_tscat_buf, const H5S_t *space, HDassert(space); HDassert(iter); HDassert(nelmts > 0); + HDassert(dxpl_cache); HDassert(buf); /* Allocate the vector I/O arrays */ @@ -364,7 +362,7 @@ done: * *------------------------------------------------------------------------- */ -static size_t +size_t H5D__gather_mem(const void *_buf, const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/) @@ -387,6 +385,7 @@ H5D__gather_mem(const void *_buf, const H5S_t *space, HDassert(space); HDassert(iter); HDassert(nelmts > 0); + HDassert(dxpl_cache); HDassert(tgath_buf); /* Allocate the vector I/O arrays */ @@ -443,8 +443,6 @@ done: hid_t H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) { - hbool_t ci_load = FALSE; /* whether MDC ci load requested */ - hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *new_file = NULL; /*file struct for new file */ hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */ hid_t ret_value; /*return value */ @@ -490,12 +488,6 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(NULL == (new_file = H5F_open(filename, flags, fcpl_id, fapl_id, dxpl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create file") - /* Check to see if both SWMR and cache image are requested. Fail if so */ - if(H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MDC cache image status") - if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE))) - HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and cache image") - /* Get an atom for the file */ if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file") @@ -554,8 +546,6 @@ done: hid_t H5Fopen(const char *filename, unsigned flags, hid_t fapl_id) { - hbool_t ci_load = FALSE; /* whether MDC ci load requested */ - hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *new_file = NULL; /*file struct for new file */ hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */ hid_t ret_value; /*return value */ @@ -585,12 +575,6 @@ H5Fopen(const char *filename, unsigned flags, hid_t fapl_id) if(NULL == (new_file = H5F_open(filename, flags, H5P_FILE_CREATE_DEFAULT, fapl_id, dxpl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to open file") - /* Check to see if both SWMR and cache image are requested. Fail if so */ - if(H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MDC cache image status") - if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE))) - HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and cache image") - /* Get an atom for the file */ if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle") @@ -1633,7 +1617,7 @@ H5Fstart_swmr_write(hid_t file_id) if(file->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "file superblock version should be at least 3") - HDassert(file->shared->latest_flags == H5F_LATEST_ALL_FLAGS); + HDassert((file->shared->latest_flags | H5F_LATEST_LAYOUT_MSG) > 0); /* Should not be marked for SWMR writing mode already */ if(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) @@ -1647,6 +1631,10 @@ H5Fstart_swmr_write(hid_t file_id) if(ci_load || ci_write ) HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and MDC cache image") + /* Flush the superblock extension */ + if(H5F_flush_tagged_metadata(file, file->shared->sblock->ext_addr, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush superblock extension") + /* Flush data buffers */ if(H5F__flush(file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") diff --git a/src/H5FDcore.c b/src/H5FDcore.c index b980b7e..2ab04dc 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -598,7 +598,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr { int o_flags; H5FD_core_t *file = NULL; - H5FD_core_fapl_t *fa = NULL; + const H5FD_core_fapl_t *fa = NULL; H5P_genplist_t *plist; /* Property list pointer */ #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; @@ -620,7 +620,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr HDassert(H5P_DEFAULT != fapl_id); if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if(NULL == (fa = (H5FD_core_fapl_t *)H5P_peek_driver_info(plist))) + if(NULL == (fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist))) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") /* Build the open flags */ @@ -638,7 +638,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr ((file_image_info.buffer == NULL) && (file_image_info.size == 0))); HDmemset(&sb, 0, sizeof(sb)); if((file_image_info.buffer != NULL) && !(H5F_ACC_CREAT & flags)) { - if(HDopen(name, o_flags, 0666) >= 0) + if(HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW) >= 0) HGOTO_ERROR(H5E_FILE, H5E_FILEEXISTS, NULL, "file already exists") /* If backing store is requested, create and stat the file @@ -646,7 +646,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr * technically an open. */ if(fa->backing_store) { - if((fd = HDopen(name, o_flags | O_CREAT, 0666)) < 0) + if((fd = HDopen(name, o_flags | O_CREAT, H5_POSIX_CREATE_MODE_RW)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create file") if(HDfstat(fd, &sb) < 0) HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file") @@ -656,7 +656,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr * store is off is when the backing_store flag is off and H5F_ACC_CREAT is * on. */ else if(fa->backing_store || !(H5F_ACC_CREAT & flags)) { - if((fd = HDopen(name, o_flags, 0666)) < 0) + if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file") if(HDfstat(fd, &sb) < 0) HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file") diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 7a4c99c..811ea8e 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -487,7 +487,7 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd o_flags |= O_DIRECT; /* Open the file */ - if ((fd=HDopen(name, o_flags, 0666))<0) + if ((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW))<0) HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file") if (HDfstat(fd, &sb)<0) diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 7c6bbd4..7c2297a 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -520,7 +520,7 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HDgettimeofday(&timeval_start, NULL); #endif /* H5_HAVE_GETTIMEOFDAY */ /* Open the file */ - if((fd = HDopen(name, o_flags, 0666)) < 0) { + if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) { int myerrno = errno; HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags); diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c index 16be455..98e1b1a 100644 --- a/src/H5FDmpi.c +++ b/src/H5FDmpi.c @@ -315,6 +315,11 @@ H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info, MPI_Comm *comm_new, MPI_Inf info_dup = info; } + /* Set MPI_ERRORS_RETURN on comm_dup so that MPI failures are not fatal, + and return codes can be checked and handled. May 23, 2017 FTW */ + if (MPI_SUCCESS != (mpi_code = MPI_Comm_set_errhandler(comm_dup, MPI_ERRORS_RETURN))) + HMPI_GOTO_ERROR(FAIL, "MPI_Errhandler_set failed", mpi_code) + /* copy them to the return arguments */ *comm_new = comm_dup; *info_new = info_dup; diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index b6385fb..32aff0f 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -341,7 +341,7 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) o_flags |= O_EXCL; /* Open the file */ - if((fd = HDopen(name, o_flags, 0666)) < 0) { + if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) { int myerrno = errno; HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags); } /* end if */ diff --git a/src/H5FSint.c b/src/H5FSint.c index 1a41172..4297291 100644 --- a/src/H5FSint.c +++ b/src/H5FSint.c @@ -77,6 +77,32 @@ /*******************/ +/*------------------------------------------------------------------------- + * Function: H5FS_init + * + * Purpose: Initialize the interface in case it is unable to initialize + * itself soon enough. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Saturday, March 4, 2000 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FS_init(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + /* FUNC_ENTER() does all the work */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS_init() */ + /*------------------------------------------------------------------------- * Function: H5FS__create_flush_depend diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h index c0467a6..247d75c 100644 --- a/src/H5FSprivate.h +++ b/src/H5FSprivate.h @@ -175,6 +175,9 @@ H5FL_SEQ_EXTERN(H5FS_section_class_t); /* Library-private Function Prototypes */ /***************************************/ +/* Package initialization routine */ +H5_DLL herr_t H5FS_init(void); + /* Free space manager routines */ H5_DLL H5FS_t *H5FS_create(H5F_t *f, hid_t dxpl_id, haddr_t *fs_addr, const H5FS_create_t *fs_create, uint16_t nclasses, diff --git a/src/H5Fint.c b/src/H5Fint.c index e52d539..7371c46 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -678,8 +678,10 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t if(H5P_get(plist, H5F_ACS_LATEST_FORMAT_NAME, &latest_format) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'latest format' flag") /* For latest format or SWMR_WRITE, activate all latest version support */ - if(latest_format || (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)) + if(latest_format) f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS; + else if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) + f->shared->latest_flags |= H5F_LATEST_LAYOUT_MSG; if(H5P_get(plist, H5F_ACS_USE_MDC_LOGGING_NAME, &(f->shared->use_mdc_logging)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'use mdc logging' flag") if(H5P_get(plist, H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME, &(f->shared->start_mdc_log_on_access)) < 0) @@ -1184,6 +1186,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F_t *ret_value = NULL; /*actual return value */ char *lock_env_var = NULL;/*env var pointer */ hbool_t use_file_locking; /*read from env var */ + hbool_t ci_load = FALSE; /* whether MDC ci load requested */ + hbool_t ci_write = FALSE; /* whether MDC CI write requested */ FUNC_ENTER_NOAPI(NULL) @@ -1317,6 +1321,12 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, set_flag = TRUE; } /* end else */ + /* Check to see if both SWMR and cache image are requested. Fail if so */ + if(H5C_cache_image_status(file, &ci_load, &ci_write) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get MDC cache image status") + if((ci_load || ci_write) && (flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE))) + HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, NULL, "can't have both SWMR and cache image") + /* Retain the name the file was opened with */ file->open_name = H5MM_xstrdup(name); @@ -1456,11 +1466,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, if(H5F_INTENT(file) & H5F_ACC_SWMR_WRITE) file->shared->sblock->status_flags |= H5F_SUPER_SWMR_WRITE_ACCESS; - /* Flush the superblock */ + /* Flush the superblock & superblock extension */ if(H5F_super_dirty(file) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, NULL, "unable to mark superblock as dirty") if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, meta_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock") + if(H5F_flush_tagged_metadata(file, file->shared->sblock->ext_addr, meta_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock extension") /* Remove the file lock for SWMR_WRITE */ if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) { @@ -1493,6 +1505,7 @@ done: if((NULL == ret_value) && file) if(H5F__dest(file, meta_dxpl_id, raw_dxpl_id, FALSE) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_open() */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 7a5c126..d702506 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -309,16 +309,16 @@ struct H5F_file_t { H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */ /* File space allocation information */ - H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ hsize_t fs_threshold; /* Free space section threshold */ - hbool_t fs_persist; /* Free-space persist or not */ + hbool_t fs_persist; /* Free-space persist or not */ hbool_t use_tmp_space; /* Whether temp. file space allocation is allowed */ haddr_t tmp_addr; /* Next address to use for temp. space in the file */ - hbool_t point_of_no_return; /* flag to indicate that we can't go back and delete a freespace header when it's used up */ + hbool_t point_of_no_return; /* Flag to indicate that we can't go back and delete a freespace header when it's used up */ H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */ - haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */ - H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */ + haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */ + H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */ hbool_t first_alloc_dealloc; /* TRUE iff free space managers */ /* are persistant and have not */ /* been used accessed for either */ @@ -333,25 +333,25 @@ struct H5F_file_t { /* HADDR_UNDEF if no cache image. */ /* Free-space aggregation info */ - unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */ - H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ - H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */ - H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */ + unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */ + H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ + H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */ + H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */ /* Paged aggregation info */ - hsize_t fs_page_size; /* File space page size */ - size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */ + hsize_t fs_page_size; /* File space page size */ + size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */ /* Metadata accumulator information */ - H5F_meta_accum_t accum; /* Metadata accumulator info */ + H5F_meta_accum_t accum; /* Metadata accumulator info */ /* Metadata retry info */ - unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */ - unsigned retries_nbins; /* # of bins for each retries[] */ + unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */ + unsigned retries_nbins; /* # of bins for each retries[] */ uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */ /* Object flush info */ - H5F_object_flush_t object_flush; /* Information for object flush callback */ + H5F_object_flush_t object_flush; /* Information for object flush callback */ }; /* diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 7c70a64..0c6f9cd 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -21,15 +21,15 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ +#include "H5private.h" /* Generic Functions */ #include "H5ACprivate.h" /* Metadata cache */ -#include "H5Eprivate.h" /* Error handling */ +#include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ -#include "H5FDprivate.h" /* File drivers */ -#include "H5Iprivate.h" /* IDs */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Iprivate.h" /* IDs */ #include "H5MFprivate.h" /* File memory management */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Pprivate.h" /* Property lists */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ #include "H5SMprivate.h" /* Shared Object Header Messages */ @@ -158,7 +158,7 @@ H5F_super_ext_open(H5F_t *f, haddr_t ext_addr, H5O_loc_t *ext_ptr) /* Open the superblock extension object header */ if(H5O_open(ext_ptr) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to open superblock extension") + HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to open superblock extension") done: FUNC_LEAVE_NOAPI(ret_value) @@ -224,12 +224,12 @@ done: /*------------------------------------------------------------------------- * Function: H5F__update_super_ext_driver_msg * - * Purpose: Update the superblock extension file driver info message if - * we are using a V 2 superblock. Observe that the function - * is a NO-OP if the file driver info message does not exist. + * Purpose: Update the superblock extension file driver info message if + * we are using a V 2 superblock. Observe that the function + * is a NO-OP if the file driver info message does not exist. * This is necessary, as the function is called whenever the - * EOA is updated, and were it to create the file driver info - * message, it would find itself in an infinite recursion. + * EOA is updated, and were it to create the file driver info + * message, it would find itself in an infinite recursion. * * Return: Success: SUCCEED * Failure: FAIL @@ -267,7 +267,7 @@ H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id) /* Check for driver info */ H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t); - /* Nothing to do unless there is both driver info and + /* Nothing to do unless there is both driver info and * the driver info superblock extension message has * already been created. */ @@ -330,9 +330,13 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial unsigned sblock_flags = H5AC__NO_FLAGS_SET; /* flags used in superblock unprotect call */ haddr_t super_addr; /* Absolute address of superblock */ haddr_t eof; /* End of file address */ - unsigned rw_flags; /* Read/write permissions for file */ - hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */ + unsigned rw_flags; /* Read/write permissions for file */ + hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */ herr_t ret_value = SUCCEED; /* Return value */ +#ifdef H5_HAVE_PARALLEL + int mpi_rank = 0, mpi_size = 1; + int mpi_result; +#endif /* H5_HAVE_PARALLEL */ FUNC_ENTER_PACKAGE_TAG(meta_dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL) @@ -354,8 +358,51 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") /* Find the superblock */ - if(H5FD_locate_signature(&fdio_info, &super_addr) < 0) - HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") +#ifdef H5_HAVE_PARALLEL + if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { + + if((mpi_rank = H5F_mpi_get_rank(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Can't get MPI rank") + + if((mpi_size = H5F_mpi_get_size(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") + } + + /* If we are an MPI application with at least two processes, the + * following superblock signature location optimization is applicable. + * + * Note:: For parallel applications which don't setup for using the + * HDF5 MPIO driver, we will arrive here with mpi_size == 1. + * This occurs because of the variable initialization (above) and the + * fact that we have skipped actually calling MPI functions to determine + * our MPI rank and size. + */ + if ( mpi_size > 1 ) { + MPI_Comm this_comm = MPI_COMM_NULL; + + if ( mpi_rank == 0 ) { + if (H5FD_locate_signature(&fdio_info, &super_addr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") + } + HDassert(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)); + + if ( MPI_COMM_NULL == (this_comm = H5F_mpi_get_comm(f)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator") + + if ( MPI_SUCCESS != + (mpi_result = MPI_Bcast(&super_addr,sizeof(super_addr), MPI_BYTE, 0, this_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + } + else { + /* Locate the signature as per per the serial library */ +#endif /* H5_HAVE_PARALLEL */ + + if (H5FD_locate_signature(&fdio_info, &super_addr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") + +#ifdef H5_HAVE_PARALLEL + } +#endif /* H5_HAVE_PARALLEL */ if(HADDR_UNDEF == super_addr) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "file signature not found") @@ -406,12 +453,12 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock") if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) - if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3) - HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE") + if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3) + HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE") /* Enable all latest version support when file has v3 superblock */ if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) - f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS; + f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS; /* Pin the superblock in the cache */ if(H5AC_pin_protected_entry(sblock) < 0) @@ -511,15 +558,15 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial * been flushed to disk by the SWMR writer process. */ if(H5F_INTENT(f) & H5F_ACC_SWMR_READ) { - /* - * When the file is opened for SWMR read access, skip the check if: - * --the file is already marked for SWMR writing and - * --the file has version 3 superblock for SWMR support - */ - if((sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) && + /* + * When the file is opened for SWMR read access, skip the check if: + * --the file is already marked for SWMR writing and + * --the file has version 3 superblock for SWMR support + */ + if((sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) && (sblock->status_flags & H5F_SUPER_WRITE_ACCESS) && sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) - skip_eof_check = TRUE; + skip_eof_check = TRUE; } /* end if */ if(!skip_eof_check && initial_read) { if(HADDR_UNDEF == (eof = H5FD_get_eof(f->shared->lf, H5FD_MEM_DEFAULT))) @@ -593,7 +640,7 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial H5O_loc_t ext_loc; /* "Object location" for superblock extension */ H5O_btreek_t btreek; /* v1 B-tree 'K' value message from superblock extension */ H5O_drvinfo_t drvinfo; /* Driver info message from superblock extension */ - size_t u; /* Local index variable */ + size_t u; /* Local index variable */ htri_t status; /* Status for message existing */ /* Sanity check - superblock extension should only be defined for @@ -614,7 +661,7 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial } /* end if */ /* Open the superblock extension */ - if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) + if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") /* Check for the extension having a 'driver info' message */ @@ -637,8 +684,8 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial /* Reset driver info message */ H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo); - HDassert(FALSE == f->shared->drvinfo_sb_msg_exists); - f->shared->drvinfo_sb_msg_exists = TRUE; + HDassert(FALSE == f->shared->drvinfo_sb_msg_exists); + f->shared->drvinfo_sb_msg_exists = TRUE; } /* end else */ } /* end if */ @@ -764,37 +811,37 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial } /* end if not marked "unknown" */ } /* end if */ - /* Check for the extension having a 'metadata cache image' message */ + /* Check for the extension having a 'metadata cache image' message */ if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { - hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0); - H5O_mdci_t mdci_msg; + hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0); + H5O_mdci_t mdci_msg; - /* if the metadata cache image superblock extension message exists, + /* if the metadata cache image superblock extension message exists, * read its contents and pass the data on to the metadata cache. * Given this data, the cache will load and decode the metadata - * cache image block, decoded it and load its contents into the - * the cache on the test protect call. + * cache image block, decoded it and load its contents into the + * the cache on the test protect call. * * Further, if the file is opened R/W, the metadata cache will - * delete the metadata cache image superblock extension and free - * the cache image block. Don't do this now as f->shared - * is not fully setup, which complicates matters. + * delete the metadata cache image superblock extension and free + * the cache image block. Don't do this now as f->shared + * is not fully setup, which complicates matters. */ /* Retrieve the 'metadata cache image message' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id)) + if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get metadata cache image message") /* Indicate to the cache that there's an image to load on first protect call */ if(H5AC_load_cache_image_on_next_protect(f, mdci_msg.addr, mdci_msg.size, rw) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "call to H5AC_load_cache_image_on_next_protect failed"); + HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "call to H5AC_load_cache_image_on_next_protect failed"); } /* end if */ /* Close superblock extension */ if(H5F_super_ext_close(f, &ext_loc, meta_dxpl_id, FALSE) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") } /* end if */ /* Update the driver info if VFD indicated to do so */ @@ -974,6 +1021,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Bump superblock version if latest superblock version support is enabled */ if(H5F_USE_LATEST_FLAGS(f, H5F_LATEST_SUPERBLOCK)) super_vers = HDF5_SUPERBLOCK_VERSION_LATEST; + /* Bump superblock version to use version 3 superblock for SWMR writing */ + else if((H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)) + super_vers = HDF5_SUPERBLOCK_VERSION_3; /* Bump superblock version to create superblock extension for SOHM info */ else if(f->shared->sohm_nindexes > 0) super_vers = HDF5_SUPERBLOCK_VERSION_2; @@ -451,9 +451,9 @@ H5I_nmembers(H5I_type_t type) FUNC_ENTER_NOAPI(FAIL) if(type <= H5I_BADID || type >= H5I_next_type) - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number") + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number") if(NULL == (type_ptr = H5I_id_type_list_g[type]) || type_ptr->init_count <= 0) - HGOTO_DONE(0); + HGOTO_DONE(0); /* Set return value */ H5_CHECKED_ASSIGN(ret_value, int64_t, type_ptr->id_count, uint64_t); diff --git a/src/H5Ipublic.h b/src/H5Ipublic.h index 7bc3c22..c737bbe 100644 --- a/src/H5Ipublic.h +++ b/src/H5Ipublic.h @@ -30,6 +30,8 @@ * When adding types here, add a section to the 'misc19' test in test/tmisc.c * to verify that the H5I{inc|dec|get}_ref() routines work correctly with it. * + * NOTE: H5I_REFERENCE is not used by the library and has been deprecated + * with a tentative removal version of 1.12.0. (DER, July 2017) */ typedef enum H5I_type_t { H5I_UNINIT = (-2), /* uninitialized type */ @@ -40,12 +42,7 @@ typedef enum H5I_type_t { H5I_DATASPACE, /* type ID for Dataspace objects */ H5I_DATASET, /* type ID for Dataset objects */ H5I_ATTR, /* type ID for Attribute objects */ - H5I_REFERENCE, /* type ID for Reference objects - ** DEPRECATED** - H5I_REFERENCE is not used in the library and - will be removed from this enumeration in a - future major release of the library. - */ + H5I_REFERENCE, /* *DEPRECATED* type ID for Reference objects */ H5I_VFL, /* type ID for virtual file layer */ H5I_GENPROP_CLS, /* type ID for generic property list classes */ H5I_GENPROP_LST, /* type ID for generic property lists */ @@ -115,6 +115,7 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /* Local Variables */ /*******************/ + /*------------------------------------------------------------------------- * Function: H5MF_init_merge_flags diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c index 1ae902f..0b300ba 100644 --- a/src/H5MFdbg.c +++ b/src/H5MFdbg.c @@ -155,8 +155,8 @@ done: herr_t H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int indent, int fwidth) { - herr_t ret_value = SUCCEED; /* Return value */ H5F_mem_page_t type; /* Memory type for iteration */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) diff --git a/src/H5Oefl.c b/src/H5Oefl.c index 49c442f..ba7a6ee 100644 --- a/src/H5Oefl.c +++ b/src/H5Oefl.c @@ -151,7 +151,7 @@ H5O_efl_decode(H5F_t *f, hid_t dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh, if((s = (const char *)H5HL_offset_into(heap, mesg->slot[u].name_offset)) == NULL) HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "unable to get external file name") - if(*s == (char)NULL) + if(*s == (char)'\0') HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "invalid external file name") mesg->slot[u].name = H5MM_xstrdup (s); HDassert(mesg->slot[u].name); diff --git a/src/H5Oflush.c b/src/H5Oflush.c index 2d93221..9764f56 100644 --- a/src/H5Oflush.c +++ b/src/H5Oflush.c @@ -370,40 +370,40 @@ H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id, hbool_ type = H5I_get_type(oid); switch(type) { - case(H5I_GROUP): + case H5I_GROUP: /* Re-open the group */ if(NULL == (object = H5G_open(obj_loc, dxpl_id))) HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to open group") break; - case(H5I_DATATYPE): + case H5I_DATATYPE: /* Re-open the named datatype */ if(NULL == (object = H5T_open(obj_loc, dxpl_id))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTOPENOBJ, FAIL, "unable to open named datatype") break; - case(H5I_DATASET): + case H5I_DATASET: /* Re-open the dataset */ if(NULL == (object = H5D_open(obj_loc, H5P_DATASET_ACCESS_DEFAULT, dxpl_id))) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open dataset") - if(!start_swmr) /* No need to handle multiple opens when H5Fstart_swmr_write() */ - if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset") + if(!start_swmr) /* No need to handle multiple opens when H5Fstart_swmr_write() */ + if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset") break; - case(H5I_UNINIT): - case(H5I_BADID): - case(H5I_FILE): - case(H5I_DATASPACE): - case(H5I_ATTR): - case(H5I_REFERENCE): - case(H5I_VFL): - case(H5I_GENPROP_CLS): - case(H5I_GENPROP_LST): - case(H5I_ERROR_CLASS): - case(H5I_ERROR_MSG): - case(H5I_ERROR_STACK): - case(H5I_NTYPES): + case H5I_UNINIT: + case H5I_BADID: + case H5I_FILE: + case H5I_DATASPACE: + case H5I_ATTR: + case H5I_REFERENCE: + case H5I_VFL: + case H5I_GENPROP_CLS: + case H5I_GENPROP_LST: + case H5I_ERROR_CLASS: + case H5I_ERROR_MSG: + case H5I_ERROR_STACK: + case H5I_NTYPES: default: HGOTO_ERROR(H5E_ARGS, H5E_CANTRELEASE, FAIL, "not a valid file object ID (dataset, group, or datatype)") break; diff --git a/src/H5Otest.c b/src/H5Otest.c index f0deade..68462a1 100644 --- a/src/H5Otest.c +++ b/src/H5Otest.c @@ -731,7 +731,7 @@ H5O_msg_move_to_new_chunk_test(hid_t oid, unsigned msg_type) /* Allocate and initialize new chunk in the file, moving the found message */ /* (*new_idx returned from this routine is unused here) */ - if(H5O__alloc_chunk(loc->file, H5AC_ind_read_dxpl_id, oh, 40, oh->nmesgs, &found_msg, &new_idx) < 0) + if(H5O__alloc_chunk(loc->file, H5AC_ind_read_dxpl_id, oh, (curr_msg->raw_size + H5O_SIZEOF_MSGHDR_OH(oh)), oh->nmesgs, &found_msg, &new_idx) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate new object header chunk") /* Break out of loop, the message was found */ @@ -1506,7 +1506,7 @@ H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry) HDassert(page_entry); /* Retrieve the 'eoa' for the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, page_entry->type))) + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, (H5FD_mem_t)page_entry->type))) HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") /* If the starting address of the page is larger than @@ -1525,7 +1525,7 @@ H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry) fdio_info.meta_dxpl = fio_info->meta_dxpl; fdio_info.raw_dxpl = fio_info->raw_dxpl; - if(H5FD_write(&fdio_info, page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0) + if(H5FD_write(&fdio_info, (H5FD_mem_t)page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0) HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ @@ -22,135 +22,28 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ -#include "H5MMprivate.h" /* Memory management */ #include "H5PLpkg.h" /* Plugin */ -#include "H5Zprivate.h" /* Filter pipeline */ /****************/ /* Local Macros */ /****************/ -#ifdef H5_HAVE_WIN32_API -#define H5PL_EXPAND_ENV_VAR { \ - long bufCharCount; \ - char *tempbuf; \ - if(NULL == (tempbuf = (char *)H5MM_malloc(H5PL_EXPAND_BUFFER_SIZE))) \ - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") \ - if((bufCharCount = ExpandEnvironmentStringsA(dl_path, tempbuf, H5PL_EXPAND_BUFFER_SIZE)) > H5PL_EXPAND_BUFFER_SIZE) { \ - tempbuf = (char *)H5MM_xfree(tempbuf); \ - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") \ - } \ - if(bufCharCount == 0) { \ - tempbuf = (char *)H5MM_xfree(tempbuf); \ - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") \ - } \ - dl_path = (char *)H5MM_xfree(dl_path); \ - dl_path = tempbuf; \ - } -#else -#define H5PL_EXPAND_ENV_VAR -#endif /* H5_HAVE_WIN32_API */ - -/****************************/ -/* Macros for supporting - * both Windows and Unix */ -/****************************/ -/* Windows support - * - * SPECIAL WINDOWS NOTE - * - * Some of the Win32 API functions expand to fooA or fooW depending on - * whether UNICODE or _UNICODE are defined. You MUST explicitly use - * the A version of the functions to force char * behavior until we - * work out a scheme for proper Windows Unicode support. - * - * If you do not do this, people will be unable to incorporate our - * source code into their own CMake builds if they define UNICODE. - */ -#ifdef H5_HAVE_WIN32_API - -#define H5PL_PATH_SEPARATOR ";" - -/* Handle for dynamic library */ -#define H5PL_HANDLE HINSTANCE - -/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ -#define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) - -/* Get the address of a symbol in dynamic library */ -#define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N) - -/* Close dynamic library */ -#define H5PL_CLOSE_LIB(H) FreeLibrary(H) - -/* Clear error - nothing to do */ -#define H5PL_CLR_ERROR - -/* maximum size for expanding env vars */ -#define H5PL_EXPAND_BUFFER_SIZE 32767 - -typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); - -/* Unix support */ -#else /* H5_HAVE_WIN32_API */ - -#define H5PL_PATH_SEPARATOR ":" - -/* Handle for dynamic library */ -#define H5PL_HANDLE void * - -/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ -#define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY) - -/* Get the address of a symbol in dynamic library */ -#define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N) - -/* Close dynamic library */ -#define H5PL_CLOSE_LIB(H) dlclose(H) - -/* Clear error */ -#define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror()) - -typedef const void *(*H5PL_get_plugin_info_t)(void); -#endif /* H5_HAVE_WIN32_API */ - -/* Whether to preload pathnames for plugin libraries */ -#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR - -/* Special symbol to indicate no plugin loading */ -#define H5PL_NO_PLUGIN "::" /******************/ /* Local Typedefs */ /******************/ -/* Type for the list of info for opened plugin libraries */ -typedef struct H5PL_table_t { - H5PL_type_t pl_type; /* plugin type */ - int pl_id; /* ID for the plugin */ - H5PL_HANDLE handle; /* plugin handle */ -} H5PL_table_t; - /********************/ /* Local Prototypes */ /********************/ -static herr_t H5PL__init_path_table(void); -static htri_t H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info); -static htri_t H5PL__open(H5PL_type_t pl_type, char *libname, int plugin_id, const void **pl_info); -static htri_t H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info); -static herr_t H5PL__close(H5PL_HANDLE handle); - /*********************/ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ @@ -161,145 +54,42 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /* Local Variables */ /*******************/ -/* Table for opened plugin libraries */ -static size_t H5PL_table_alloc_g = 0; -static size_t H5PL_table_used_g = 0; -static H5PL_table_t *H5PL_table_g = NULL; - -/* Table of location paths for plugin libraries */ -static char *H5PL_path_table_g[H5PL_MAX_PATH_NUM]; -static size_t H5PL_num_paths_g = 0; -static hbool_t H5PL_path_found_g = FALSE; - -/* Enable all plugin libraries */ -static unsigned int H5PL_plugin_g = H5PL_ALL_PLUGIN; - - - -/*-------------------------------------------------------------------------- -NAME - H5PL__init_package -- Initialize interface-specific information -USAGE - herr_t H5PL__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5PL__init_package(void) -{ - char *preload_path; - - FUNC_ENTER_PACKAGE_NOERR - - /* Retrieve pathnames from HDF5_PLUGIN_PRELOAD if the user sets it - * to tell the library to load plugin libraries without search. - */ - if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD"))) - /* Special symbol "::" means no plugin during data reading. */ - if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN)) - H5PL_plugin_g = 0; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5PL__init_package() */ - /*------------------------------------------------------------------------- - * Function: H5PL_term_package + * Function: H5PLset_loading_state * - * Purpose: Terminate the H5PL interface: release all memory, reset all - * global variables to initial values. This only happens if all - * types have been destroyed from other interfaces. + * Purpose: Control the loading of dynamic plugin types. * - * Return: Success: Positive if any action was taken that might - * affect some other interface; zero otherwise. - * Failure: Negative. + * The plugin_control_mask parameter is a bitfield that controls + * whether certain classes of plugins (e.g.: filters, + * VOL drivers) will be loaded by the library. * - * Programmer: Raymond Lu - * 20 February 2013 + * plugin bit = 0, will prevent the use of that dynamic plugin type. + * plugin bit = 1, will allow the use of that dynamic plugin type. * - *------------------------------------------------------------------------- - */ -int -H5PL_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if(H5_PKG_INIT_VAR) { - size_t u; /* Local index variable */ - - /* Close opened dynamic libraries */ - if(H5PL_table_g) { - for(u = 0; u < H5PL_table_used_g; u++) - H5PL__close((H5PL_table_g[u]).handle); - - /* Free the table of dynamic libraries */ - H5PL_table_g = (H5PL_table_t *)H5MM_xfree(H5PL_table_g); - H5PL_table_used_g = H5PL_table_alloc_g = 0; - - n++; - } /* end if */ - - /* Free the table of search paths */ - if(H5PL_num_paths_g > 0) { - for(u = 0; u < H5PL_num_paths_g; u++) - if(H5PL_path_table_g[u]) - H5PL_path_table_g[u] = (char *)H5MM_xfree(H5PL_path_table_g[u]); - H5PL_num_paths_g = 0; - H5PL_path_found_g = FALSE; - - n++; - } /* end if */ - - /* Mark the interface as uninitialized */ - if(0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ - - FUNC_LEAVE_NOAPI(n) -} /* end H5PL_term_package() */ - - -/*------------------------------------------------------------------------- - * Function: H5PLset_loading_state - * - * Purpose: Control the loading of dynamic plugin types. - * - * This function will not allow plugin types if the pathname from the HDF5_PLUGIN_PRELOAD - * environment variable is set to the special "::" string. + * A list of pre-defined masks can be found in H5PLpublic.h. + * Set the mask to 0 to disable all plugins. * - * plugin bit = 0, will prevent the use of that dynamic plugin type. - * plugin bit = 1, will allow the use of that dynamic plugin type. + * This function will not allow plugin types if the pathname + * from the HDF5_PLUGIN_PRELOAD environment variable is set to + * the special "::" string. * - * H5PL_TYPE_FILTER changes just dynamic filters - * A H5PL_ALL_PLUGIN will enable all dynamic plugin types - * A zero value will disable all dynamic plugin types - * - * Return: Non-negative or success + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLset_loading_state(unsigned int plugin_type) +H5PLset_loading_state(unsigned int plugin_control_mask) { - char *preload_path; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "Iu", plugin_type); - - /* change the bit value of the requested plugin type(s) */ - H5PL_plugin_g = plugin_type; + H5TRACE1("e", "Iu", plugin_control_mask); - /* check if special ENV variable is set and disable all plugin types */ - if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD"))) - /* Special symbol "::" means no plugin during data reading. */ - if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN)) - H5PL_plugin_g = 0; + /* Set the plugin control mask */ + if(H5PL__set_plugin_control_mask(plugin_control_mask) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTSET, FAIL, "error setting plugin control mask") done: FUNC_LEAVE_API(ret_value) @@ -307,27 +97,35 @@ done: /*------------------------------------------------------------------------- - * Function: H5PLget_loading_state + * Function: H5PLget_loading_state * - * Purpose: Query state of the loading of dynamic plugin types. + * Purpose: Get the bitmask that controls whether certain classes + * of plugins (e.g.: filters, VOL drivers) will be loaded + * by the library. * - * This function will return the state of the global flag. + * Zero if all plugin types are disabled + * Negative if all plugin types are enabled + * Positive if one or more of the plugin types are enabled * - * Return: Zero if all plugin types are disabled, negative if all - * plugin types are enabled, positive if one or more of the plugin types are enabled. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLget_loading_state(unsigned int *plugin_type) +H5PLget_loading_state(unsigned int *plugin_control_mask) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*Iu", plugin_type); + H5TRACE1("e", "*Iu", plugin_control_mask); + + if (NULL == plugin_control_mask) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_control_mask parameter cannot be NULL") - if(plugin_type) - *plugin_type = H5PL_plugin_g; + /* Set the plugin control mask */ + if(H5PL__get_plugin_control_mask(plugin_control_mask) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "error getting plugin control mask") done: FUNC_LEAVE_API(ret_value) @@ -335,674 +133,282 @@ done: /*------------------------------------------------------------------------- - * Function: H5PL_load - * - * Purpose: Given the plugin type and identifier, this function searches - * and/or loads a dynamic plugin library first among the already - * opened libraries then in the designated location paths. - * - * Return: Non-NULL on success/NULL on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -const void * -H5PL_load(H5PL_type_t type, int id) -{ - htri_t found; /* Whether the plugin was found */ - const void *plugin_info = NULL; - const void *ret_value = NULL; - - FUNC_ENTER_NOAPI(NULL) - - switch(type) { - case H5PL_TYPE_FILTER: - if((H5PL_plugin_g & H5PL_FILTER_PLUGIN) == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id) - break; - - case H5PL_TYPE_ERROR: - case H5PL_TYPE_NONE: - default: - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id) - } /* end switch */ - - /* Initialize the location paths for dynamic libraries, if they aren't - * already set up. - */ - if(FALSE == H5PL_path_found_g) - if(H5PL__init_path_table() < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, NULL, "can't initialize search path table") - - /* Search in the table of already loaded plugin libraries */ - if((found = H5PL__search_table(type, id, &plugin_info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in table failed") - - /* If not found, iterate through the path table to find the right dynamic library */ - if(!found) { - size_t i; /* Local index variable */ - - for(i = 0; i < H5PL_num_paths_g; i++) { - if((found = H5PL__find(type, id, H5PL_path_table_g[i], &plugin_info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in paths failed") - - /* Break out if found */ - if(found) { - HDassert(plugin_info); - break; - } /* end if */ - } /* end for */ - } /* end if */ - - /* Check if we found the plugin */ - if(found) - ret_value = plugin_info; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL_load() */ - - -/*------------------------------------------------------------------------- - * Function: H5PLappend + * Function: H5PLappend * - * Purpose: Insert a plugin path at the end of the list. + * Purpose: Insert a plugin search path at the end of the list. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLappend(const char *plugin_path) +H5PLappend(const char *search_path) { herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*s", plugin_path); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE1("e", "*s", search_path); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - H5PL_path_table_g[H5PL_num_paths_g] = dl_path; - H5PL_num_paths_g++; + /* Append the search path to the path table */ + if (H5PL__append_path(search_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTAPPEND, FAIL, "unable to append search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLappend() */ - + /*------------------------------------------------------------------------- - * Function: H5PLprepend + * Function: H5PLprepend * - * Purpose: Insert a plugin path at the beginning of the list. + * Purpose: Insert a plugin search path at the beginning of the list. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLprepend(const char *plugin_path) +H5PLprepend(const char *search_path) { herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; - unsigned int plindex; FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*s", plugin_path); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE1("e", "*s", search_path); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - for (plindex = (unsigned int)H5PL_num_paths_g; plindex > 0; plindex--) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1]; - H5PL_path_table_g[0] = dl_path; - H5PL_num_paths_g++; + /* Prepend the search path to the path table */ + if (H5PL__prepend_path(search_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLprepend() */ - + /*------------------------------------------------------------------------- - * Function: H5PLreplace + * Function: H5PLreplace * - * Purpose: Replace the path at the specified index. + * Purpose: Replace the path at the specified index. The path at the + * index must exist. * - * Return: Non-negative or success. + * Return: Non-negative or success. * *------------------------------------------------------------------------- */ herr_t H5PLreplace(const char *plugin_path, unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE2("e", "*sIu", plugin_path, index); - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - if(H5PL_path_table_g[index]) - H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]); - H5PL_path_table_g[index] = dl_path; + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Insert the search path into the path table */ + if (H5PL__replace_path(search_path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLreplace() */ - + /*------------------------------------------------------------------------- - * Function: H5PLinsert + * Function: H5PLinsert * - * Purpose: Insert a plugin path at the specified index, moving other paths after the index. + * Purpose: Insert a plugin search path at the specified index, moving + * other paths after the index. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLinsert(const char *plugin_path, unsigned int index) +H5PLinsert(const char *search_path, unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; - unsigned int plindex; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE2("e", "*sIu", plugin_path, index); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - - H5PL_EXPAND_ENV_VAR - - for(plindex = (unsigned int)H5PL_num_paths_g; plindex > index; plindex--) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1]; - H5PL_path_table_g[index] = dl_path; - H5PL_num_paths_g++; + H5TRACE2("e", "*sIu", search_path, index); + + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if ((0 != num_paths) && (index >= num_paths)) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Insert the search path into the path table */ + if (H5PL__insert_path(search_path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLinsert() */ - + /*------------------------------------------------------------------------- - * Function: H5PLremove + * Function: H5PLremove * - * Purpose: Remove the plugin path at the specifed index and compacting the list. + * Purpose: Remove the plugin path at the specifed index and compact + * the list. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative + * + * Return: Non-negative or success. * *------------------------------------------------------------------------- */ herr_t H5PLremove(unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - unsigned int plindex; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "Iu", index); - if(H5PL_num_paths_g == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == H5PL_path_table_g[index]) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index") - H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]); - - H5PL_num_paths_g--; - for(plindex = index; plindex < (unsigned int)H5PL_num_paths_g; plindex++) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex + 1]; - H5PL_path_table_g[H5PL_num_paths_g] = NULL; + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Delete the search path from the path table */ + if (H5PL__remove_path(index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "unable to remove search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLremove() */ - + /*------------------------------------------------------------------------- - * Function: H5PLget + * Function: H5PLget + * + * Purpose: Query the plugin path at a specified index. * - * Purpose: Query the plugin path at the specified index. + * If 'path_buf' is non-NULL then up to 'buf_size' bytes will be written into + * that buffer and the length of the path name will be returned. * - * Return: Success: The length of path. + * If 'path_buf' is NULL, this function will simply return the number of + * characters required to store the path name, ignoring 'path_buf' and + * 'buf_size' * - * If `pathname' is non-NULL then write up to `size' bytes into that - * buffer and always return the length of the pathname. - * Otherwise `size' is ignored and the function does not store the pathname, - * just returning the number of characters required to store the pathname. - * If an error occurs then the buffer pointed to by `pathname' (NULL or non-NULL) - * is unchanged and the function returns a negative value. - * If a zero is returned for the name's length, then there is no pathname - * associated with the index. + * If an error occurs then the buffer pointed to by 'path_buf' + * (NULL or non-NULL) will be unchanged and the function will return a + * negative value. + * + * If a zero is returned for the name's length, then there is no path name + * associated with the index and the 'path_buf' buffer will be unchanged. + * + * Return: Success: The length of path + * Failure: A negative value * *------------------------------------------------------------------------- */ ssize_t -H5PLget(unsigned int index, char *pathname/*out*/, size_t size) +H5PLget(unsigned int index, char *path_buf, size_t buf_size) { - ssize_t ret_value = 0; /* Return value */ - size_t len = 0; /* Length of pathname */ - char *dl_path = NULL; + unsigned num_paths; /* Current number of stored paths */ + const char *path = NULL; /* path from table */ + size_t path_len = 0; /* Length of path */ + ssize_t ret_value = 0; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("Zs", "Iuxz", index, pathname, size); - if(H5PL_num_paths_g == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5PL_path_table_g[index])) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index") - len = HDstrlen(dl_path); - if(pathname) { - HDstrncpy(pathname, dl_path, MIN((size_t)(len + 1), size)); - if((size_t)len >= size) - pathname[size - 1] = '\0'; + H5TRACE3("Zs", "Iu*sz", index, path_buf, buf_size); + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Check if the search table is empty */ + if (H5PL__get_num_paths() == 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, (-1), "plugin search path table is empty") + + /* Get the path at the specified index and its length */ + if (NULL == (path = H5PL__get_path(index))) + HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, (-1), "no path stored at that index") + path_len = HDstrlen(path); + + /* If the path buffer is not NULL, copy the path to the buffer */ + if (path_buf) { + HDstrncpy(path_buf, path, MIN((size_t)(path_len + 1), buf_size)); + if ((size_t)path_len >= buf_size) + path_buf[buf_size - 1] = '\0'; } /* end if */ /* Set return value */ - ret_value = (ssize_t)len; + ret_value = (ssize_t)path_len; done: FUNC_LEAVE_API(ret_value) } /* end H5PLget() */ - + /*------------------------------------------------------------------------- - * Function: H5PLsize + * Function: H5PLsize * - * Purpose: Query the size of the current list of plugin paths. + * Purpose: Get the number of stored plugin paths. + * XXX: This is a terrible name. Can it be changed? * - * Return: Plugin path size + * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ herr_t -H5PLsize(unsigned int *listsize) +H5PLsize(unsigned int *num_paths) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*Iu", listsize); + H5TRACE1("e", "*Iu", num_paths); + + /* Check arguments */ + if (!num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "num_paths parameter cannot be NULL") - *listsize = (unsigned int)H5PL_num_paths_g; + /* Get the number of stored plugin paths */ + *num_paths = H5PL__get_num_paths(); done: FUNC_LEAVE_API(ret_value) } /* end H5PLsize() */ - -/*------------------------------------------------------------------------- - * Function: H5PL__init_path_table - * - * Purpose: Initialize the path table. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * 18 March 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5PL__init_path_table(void) -{ - char *dl_path = NULL; - char *origin_dl_path; - char *dir; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it - * or from the default paths if it isn't set. - */ - origin_dl_path = HDgetenv("HDF5_PLUGIN_PATH"); - if(NULL == origin_dl_path) - dl_path = H5MM_strdup(H5PL_DEFAULT_PATH); - else - dl_path = H5MM_strdup(origin_dl_path); - if(NULL == dl_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - - H5PL_EXPAND_ENV_VAR - - /* Put paths in the path table. They are separated by ":" */ - dir = HDstrtok(dl_path, H5PL_PATH_SEPARATOR); - while(dir) { - /* Check for too many directories in path */ - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == (H5PL_path_table_g[H5PL_num_paths_g] = H5MM_strdup(dir))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - H5PL_num_paths_g++; - dir = HDstrtok(NULL, H5PL_PATH_SEPARATOR); - } /* end while */ - - H5PL_path_found_g = TRUE; - -done: - if(dl_path) - dl_path = (char *)H5MM_xfree(dl_path); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__init_path_table() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__find - * - * Purpose: Given a path, this function opens the directory and envokes - * another function to go through all files to find the right - * plugin library. Two function definitions are for Unix and - * Windows. - * - * Return: TRUE on success, - * FALSE on not found, - * negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -#ifndef H5_HAVE_WIN32_API -static htri_t -H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info) -{ - char *pathname = NULL; - DIR *dirp = NULL; - struct dirent *dp; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Open the directory */ - if(!(dirp = HDopendir(dir))) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir) - - /* Iterates through all entries in the directory to find the right plugin library */ - while(NULL != (dp = HDreaddir(dirp))) { - /* The library we are looking for should be called libxxx.so... on Unix - * or libxxx.xxx.dylib on Mac. - */ -#ifndef __CYGWIN__ - if(!HDstrncmp(dp->d_name, "lib", (size_t)3) && - (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) { -#else - if(!HDstrncmp(dp->d_name, "cyg", (size_t)3) && - HDstrstr(dp->d_name, ".dll") ) { - -#endif - h5_stat_t my_stat; - size_t pathname_len; - htri_t found_in_dir; - - /* Allocate & initialize the path name */ - pathname_len = HDstrlen(dir) + HDstrlen(dp->d_name) + 2; - if(NULL == (pathname = (char *)H5MM_malloc(pathname_len))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - HDsnprintf(pathname, pathname_len, "%s/%s", dir, dp->d_name); - - /* Get info for directory entry */ - if(HDstat(pathname, &my_stat) == -1) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file: %s", HDstrerror(errno)) - - /* If it is a directory, skip it */ - if(S_ISDIR(my_stat.st_mode)) - continue; - - /* Attempt to open the dynamic library as a filter library */ - if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") - if(found_in_dir) - HGOTO_DONE(TRUE) /* Indicate success */ - pathname = (char *)H5MM_xfree(pathname); - } /* end if */ - } /* end while */ - -done: - if(dirp) - if(HDclosedir(dirp) < 0) - HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno)) - pathname = (char *)H5MM_xfree(pathname); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__find() */ -#else /* H5_HAVE_WIN32_API */ -static htri_t -H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info) -{ - WIN32_FIND_DATAA fdFile; - HANDLE hFind; - char *pathname = NULL; - char service[2048]; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Specify a file mask. *.* = We want everything! */ - sprintf(service, "%s\\*.dll", dir); - if((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory") - - do { - /* Find first file will always return "." - * and ".." as the first two directories. - */ - if(HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) { - size_t pathname_len; - htri_t found_in_dir; - - /* Allocate & initialize the path name */ - pathname_len = HDstrlen(dir) + HDstrlen(fdFile.cFileName) + 2; - if(NULL == (pathname = (char *)H5MM_malloc(pathname_len))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - HDsnprintf(pathname, pathname_len, "%s\\%s", dir, fdFile.cFileName); - - /* Is the entity a File or Folder? */ - if(fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - continue; - - if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") - if(found_in_dir) - HGOTO_DONE(TRUE) /* Indicate success */ - pathname = (char *)H5MM_xfree(pathname); - } /* end if */ - } while(FindNextFileA(hFind, &fdFile)); /* Find the next file. */ - -done: - if(hFind) - FindClose(hFind); - if(pathname) - pathname = (char *)H5MM_xfree(pathname); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__find() */ -#endif /* H5_HAVE_WIN32_API */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__open - * - * Purpose: Iterates through all files to find the right plugin library. - * It loads the dynamic plugin library and keeps it on the list - * of loaded libraries. - * - * Return: TRUE on success, - * FALSE on not found, - * negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static htri_t -H5PL__open(H5PL_type_t pl_type, char *libname, int pl_id, const void **pl_info) -{ - H5PL_HANDLE handle = NULL; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* There are different reasons why a library can't be open, e.g. wrong architecture. - * simply continue if we can't open it. - */ - if(NULL == (handle = H5PL_OPEN_DLIB(libname))) { - H5PL_CLR_ERROR; /* clear error */ - } /* end if */ - else { - H5PL_get_plugin_info_t get_plugin_info = NULL; - - /* Return a handle for the function H5PLget_plugin_info in the dynamic library. - * The plugin library is suppose to define this function. - */ - if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) { - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - } /* end if */ - else { - const H5Z_class2_t *plugin_info; - - /* Invoke H5PLget_plugin_info to verify this is the right library we are looking for. - * Move on if it isn't. - */ - if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) { - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") - } /* end if */ - - /* Successfully found plugin library, check if it's the right one */ - if(plugin_info->id == pl_id) { - /* Expand the table if it is too small */ - if(H5PL_table_used_g >= H5PL_table_alloc_g) { - size_t n = MAX(H5Z_MAX_NFILTERS, 2 * H5PL_table_alloc_g); - H5PL_table_t *table = (H5PL_table_t *)H5MM_realloc(H5PL_table_g, n * sizeof(H5PL_table_t)); - - if(!table) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to extend dynamic library table") - - H5PL_table_g = table; - H5PL_table_alloc_g = n; - } /* end if */ - - (H5PL_table_g[H5PL_table_used_g]).handle = handle; - (H5PL_table_g[H5PL_table_used_g]).pl_type = pl_type; - (H5PL_table_g[H5PL_table_used_g]).pl_id = plugin_info->id; - H5PL_table_used_g++; - - /* Set the plugin info to return */ - *pl_info = (const void *)plugin_info; - - /* Indicate success */ - ret_value = TRUE; - } /* end if */ - else - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - } /* end if */ - } /* end else */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__open() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__search_table - * - * Purpose: Search in the list of already opened dynamic libraries - * to see if the one we are looking for is already opened. - * - * Return: TRUE on success, - * FALSE on not found, - * Negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static htri_t -H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info) -{ - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Search in the table of already opened dynamic libraries */ - if(H5PL_table_used_g > 0) { - size_t i; - - for(i = 0; i < H5PL_table_used_g; i++) { - if((plugin_type == (H5PL_table_g[i]).pl_type) && (type_id == (H5PL_table_g[i]).pl_id)) { - H5PL_get_plugin_info_t get_plugin_info; - const H5Z_class2_t *plugin_info; - - if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_table_g[i]).handle, "H5PLget_plugin_info"))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") - - if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info") - - *info = plugin_info; - HGOTO_DONE(TRUE) - } /* end if */ - } /* end for */ - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__search_table() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__close - * - * Purpose: Closes the handle for dynamic library - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5PL__close(H5PL_HANDLE handle) -{ - FUNC_ENTER_STATIC_NOERR - - H5PL_CLOSE_LIB(handle); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5PL__close() */ - diff --git a/src/H5PLint.c b/src/H5PLint.c new file mode 100644 index 0000000..c887f86 --- /dev/null +++ b/src/H5PLint.c @@ -0,0 +1,384 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Internal routines for managing plugins. + * + */ + + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ +#include "H5Zprivate.h" /* Filter pipeline */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Bitmask that controls whether classes of plugins + * (e.g.: filters, VOL drivers) can be loaded. + */ +static unsigned int H5PL_plugin_control_mask_g = H5PL_ALL_PLUGIN; + +/* This flag will be set to FALSE if the HDF5_PLUGIN_PRELOAD + * environment variable was set to H5PL_NO_PLUGIN at + * package initialization. + */ +static hbool_t H5PL_allow_plugins_g = TRUE; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_plugin_control_mask + * + * Purpose: Gets the internal plugin control mask value. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__get_plugin_control_mask(unsigned int *mask /*out*/) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args - Just assert on package functions */ + HDassert(mask); + + /* Return the mask */ + *mask = H5PL_plugin_control_mask_g; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__get_plugin_control_mask() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__set_plugin_control_mask + * + * Purpose: Sets the internal plugin control mask value. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__set_plugin_control_mask(unsigned int mask) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Only allow setting this if plugins have not been disabled. + * XXX: Note that we don't consider this an error, but instead + * silently ignore it. We may want to consider this behavior + * more carefully. + */ + if (H5PL_allow_plugins_g) + H5PL_plugin_control_mask_g = mask; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__set_plugin_control_mask() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__init_package + * + * Purpose: Initialize any package-specific data and call any init + * routines for the package. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__init_package(void) +{ + char *env_var = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the environment variable to determine if the user wants + * to ignore plugins. The special symbol H5PL_NO_PLUGIN (defined in + * H5PLpublic.h) means we don't want to load plugins. + */ + if (NULL != (env_var = HDgetenv("HDF5_PLUGIN_PRELOAD"))) + if (!HDstrcmp(env_var, H5PL_NO_PLUGIN)) { + H5PL_plugin_control_mask_g = 0; + H5PL_allow_plugins_g = FALSE; + } + + /* Create the table of previously-loaded plugins */ + if (H5PL__create_plugin_cache() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin cache") + + /* Create the table of search paths for dynamic libraries */ + if (H5PL__create_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin search path table") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL_term_package + * + * Purpose: Terminate the H5PL interface: release all memory, reset all + * global variables to initial values. This only happens if all + * types have been destroyed from other interfaces. + * + * Return: Success: Positive if any action was taken that might + * affect some other interface; zero otherwise + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +int +H5PL_term_package(void) +{ + hbool_t already_closed = FALSE; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT + + if (H5_PKG_INIT_VAR) { + + /* Close the plugin cache. + * We need to bump the return value if we did any real work here. + */ + if (H5PL__close_plugin_cache(&already_closed) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache") + if (!already_closed) + ret_value++; + + /* Close the search path table and free the paths */ + if (H5PL__close_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table") + + /* Mark the interface as uninitialized */ + if (0 == ret_value) + H5_PKG_INIT_VAR = FALSE; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL_load + * + * Purpose: Given the plugin type and identifier, this function searches + * for and, if found, loads a dynamic plugin library. + * + * The function searches first in the cached plugins and then + * in the paths listed in the path table. + * + * Return: Success: A pointer to the plugin info + * Failure: NULL + * + *------------------------------------------------------------------------- + */ +const void * +H5PL_load(H5PL_type_t type, int id) +{ + H5PL_search_params_t search_params; + hbool_t found = FALSE; /* Whether the plugin was found */ + const void *plugin_info = NULL; + const void *ret_value = NULL; + + FUNC_ENTER_NOAPI(NULL) + + /* Check if plugins can be loaded for this plugin type */ + switch (type) { + case H5PL_TYPE_FILTER: + if ((H5PL_plugin_control_mask_g & H5PL_FILTER_PLUGIN) == 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id) + break; + + case H5PL_TYPE_ERROR: + case H5PL_TYPE_NONE: + default: + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id) + } + + /* Set up the search parameters */ + search_params.type = type; + search_params.id = id; + + /* Search in the table of already loaded plugin libraries */ + if(H5PL__find_plugin_in_cache(&search_params, &found, &plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in plugin cache failed") + + /* If not found, try iterating through the path table to find an appropriate plugin */ + if (!found) + if (H5PL__find_plugin_in_path_table(&search_params, &found, &plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in path table failed") + + /* Set the return value we found the plugin */ + if (found) + ret_value = plugin_info; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL_load() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__open + * + * Purpose: Opens a plugin. + * + * The success parameter will be set to TRUE and the plugin_info + * parameter will be filled in on success. Otherwise, they + * will be FALSE and NULL, respectively. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +/* NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting + * object pointers to function pointers, which is undefined in ANSI C. + * This is basically unavoidable due to the nature of dlsym() and *is* + * defined in POSIX, so it's fine. + * + * This pragma only needs to surround the assignment of the + * get_plugin_info function pointer, but early (4.4.7, at least) gcc + * only allows diagnostic pragmas to be toggled outside of functions. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +herr_t +H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const void **plugin_info) +{ + H5PL_HANDLE handle = NULL; + H5PL_get_plugin_info_t get_plugin_info = NULL; + htri_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(success); + HDassert(plugin_info); + + /* Initialize out parameters */ + *success = FALSE; + *plugin_info = NULL; + + /* There are different reasons why a library can't be open, e.g. wrong architecture. + * If we can't open the library, just return. + */ + if (NULL == (handle = H5PL_OPEN_DLIB(path))) { + H5PL_CLR_ERROR; /* clear error */ + HGOTO_DONE(SUCCEED); + } + + /* Return a handle for the function H5PLget_plugin_info in the dynamic library. + * The plugin library is suppose to define this function. + */ + if (NULL != (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) { + + const H5Z_class2_t *info; + + /* Get the plugin info */ + if (NULL == (info = (const H5Z_class2_t *)(*get_plugin_info)())) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") + + /* Check if the filter IDs match */ + if (info->id == id) { + + /* Store the plugin in the cache */ + if (H5PL__add_plugin(type, id, handle)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to add new plugin to plugin cache") + + /* Set output parameters */ + *success = TRUE; + *plugin_info = (const void *)info; + } + } + +done: + if (!success && handle) + if (H5PL__close(handle) < 0) + HDONE_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__open() */ +#pragma GCC diagnostic pop + + +/*------------------------------------------------------------------------- + * Function: H5PL__close + * + * Purpose: Closes the handle for dynamic library + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close(H5PL_HANDLE handle) +{ + FUNC_ENTER_PACKAGE_NOERR + + H5PL_CLOSE_LIB(handle); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5PL__close() */ + diff --git a/src/H5PLpath.c b/src/H5PLpath.c new file mode 100644 index 0000000..435802a --- /dev/null +++ b/src/H5PLpath.c @@ -0,0 +1,776 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Code to implement a path table which stores plugin search paths. + * + * The path table is implemented as a dynamic, global array which + * will grow as new paths are inserted. The capacity of the path + * table never shrinks (though given the low number of paths + * expected and the low likelihood of paths being removed, this + * seems unlikely to be a problem). Inserts and removals rework + * the array so that there are no 'holes' in the in-use part + * of the array. + * + * Note that it's basically up to the user to manage the indexes + * when a complicated series of insert, overwrite, and, remove + * operations take place. + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Initial capacity of the path table */ +#define H5PL_INITIAL_PATH_CAPACITY 16 + +/* The amount to add to the capacity when the table is full */ +#define H5PL_PATH_CAPACITY_ADD 16 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5PL__insert_at(const char *path, unsigned int index); +static herr_t H5PL__make_space_at(unsigned int index); +static herr_t H5PL__replace_at(const char *path, unsigned int index); +static herr_t H5PL__expand_path_table(void); +static herr_t H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info); + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Stored plugin paths to search */ +static char **H5PL_paths_g = NULL; + +/* The number of stored paths */ +static unsigned H5PL_num_paths_g = 0; + +/* The capacity of the path table */ +static unsigned H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__insert_at() + * + * Purpose: Insert a path at a particular index in the path table. + * Does not clobber! Will move existing paths up to make + * room. Use H5PL__replace_at(index) if you want to clobber. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__insert_at(const char *path, unsigned int index) +{ + char *path_copy = NULL; /* copy of path string (for storing) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Expand the table if it is full */ + if (H5PL_num_paths_g == H5PL_path_capacity_g) + if (H5PL__expand_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand path table") + + /* Copy the path for storage so the caller can dispose of theirs */ + if (NULL == (path_copy = H5MM_strdup(path))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path") + +#ifdef H5_HAVE_WIN32_API + /* Clean up Microsoft Windows environment variables in the path string */ + if(H5_expand_windows_env_vars(&path_copy)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string") +#endif /* H5_HAVE_WIN32_API */ + + /* If the table entry is in use, make some space */ + if (H5PL_paths_g[index]) + if (H5PL__make_space_at(index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "unable to make space in the table for the new entry") + + /* Insert the copy of the search path into the table at the specified index */ + H5PL_paths_g[index] = path_copy; + H5PL_num_paths_g++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__insert_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__make_space_at() + * + * Purpose: Free up a slot in the path table, moving existing path + * entries as necessary. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__make_space_at(unsigned int index) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check args - Just assert on package functions */ + HDassert(index < H5PL_path_capacity_g); + + /* Copy the paths back to make a space */ + for (u = H5PL_num_paths_g; u > index; u--) + H5PL_paths_g[u] = H5PL_paths_g[u-1]; + + H5PL_paths_g[index] = NULL; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__make_space_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__replace_at() + * + * Purpose: Replace a path at a particular index in the path table. + * The path in the table must exist and will be freed by this + * function. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__replace_at(const char *path, unsigned int index) +{ + char *path_copy = NULL; /* copy of path string (for storing) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Check that the table entry is in use */ + if (!H5PL_paths_g[index]) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, FAIL, "path entry at index %u in the table is NULL", index) + + /* Copy the path for storage so the caller can dispose of theirs */ + if (NULL == (path_copy = H5MM_strdup(path))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path") + +#ifdef H5_HAVE_WIN32_API + /* Clean up Microsoft Windows environment variables in the path string */ + if (H5_expand_windows_env_vars(&path_copy)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string") +#endif /* H5_HAVE_WIN32_API */ + + /* Free the existing path entry */ + H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]); + + /* Copy the search path into the table at the specified index */ + H5PL_paths_g[index] = path_copy; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__create_path_table + * + * Purpose: Create the collection of paths that will be searched + * when loading plugins. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__create_path_table(void) +{ + char *env_var= NULL; /* Path string from environment variable */ + char *paths = NULL; /* Delimited paths string. Either from the + * environment variable or the default. + */ + char *next_path = NULL; /* A path tokenized from the paths string */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Allocate memory for the path table */ + H5PL_num_paths_g = 0; + H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY; + if (NULL == (H5PL_paths_g = (char **)H5MM_calloc((size_t)H5PL_path_capacity_g * sizeof(char *)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path table") + + /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it + * or from the default paths if it isn't set. + */ + env_var = HDgetenv("HDF5_PLUGIN_PATH"); + if (NULL == env_var) + paths = H5MM_strdup(H5PL_DEFAULT_PATH); + else + paths = H5MM_strdup(env_var); + + if (NULL == paths) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path copy") + + /* Separate the paths and store them */ + /* XXX: strtok() is not thread-safe */ + next_path = HDstrtok(paths, H5PL_PATH_SEPARATOR); + while (next_path) { + + /* Insert the path into the table */ + if (H5PL__append_path(next_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't insert path: %s", next_path) + + /* Get the next path from the environment string */ + next_path = HDstrtok(NULL, H5PL_PATH_SEPARATOR); + } /* end while */ + +done: + if (paths) + paths = (char *)H5MM_xfree(paths); + + /* Try to clean up on errors */ + if (FAIL == ret_value) { + if (H5PL_paths_g) + H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g); + H5PL_path_capacity_g = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__create_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__close_path_table + * + * Purpose: Close the collection of paths that will be searched + * when loading plugins. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close_path_table(void) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Free paths */ + for (u = 0; u < H5PL_num_paths_g; u++) + if (H5PL_paths_g[u]) + H5PL_paths_g[u] = (char *)H5MM_xfree(H5PL_paths_g[u]); + + /* Free path table */ + H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g); + + /* Reset values */ + H5PL_num_paths_g = 0; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__close_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_num_paths + * + * Purpose: Gets the number of plugin paths that have been stored. + * + * Return: Success: The number of paths + * Failture: Can't fail + *------------------------------------------------------------------------- + */ +unsigned +H5PL__get_num_paths(void) +{ + FUNC_ENTER_PACKAGE_NOERR + + FUNC_LEAVE_NOAPI(H5PL_num_paths_g) + +} /* end H5PL__get_num_paths() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__expand_path_table + * + * Purpose: Expand the path table when it's full. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__expand_path_table(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Update the capacity */ + H5PL_path_capacity_g += H5PL_PATH_CAPACITY_ADD; + + /* Resize the array */ + if(NULL == (H5PL_paths_g = (char **)H5MM_realloc(H5PL_paths_g, (size_t)H5PL_path_capacity_g * sizeof(char *)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for path table failed") + + /* Initialize the new memory */ + HDmemset(H5PL_paths_g + H5PL_num_paths_g, 0, (size_t)H5PL_PATH_CAPACITY_ADD * sizeof(char *)); + +done: + /* Set the path capacity back if there were problems */ + if (FAIL == ret_value) + H5PL_path_capacity_g -= H5PL_PATH_CAPACITY_ADD; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__expand_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__append_path + * + * Purpose: Insert a path at the end of the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__append_path(const char *path) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Insert the path at the end of the table */ + if (H5PL__insert_at(path, H5PL_num_paths_g) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to append search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__append_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__prepend_path + * + * Purpose: Insert a path at the beginning of the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__prepend_path(const char *path) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Insert the path at the beginning of the table */ + if (H5PL__insert_at(path, 0) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__prepend_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__replace_path + * + * Purpose: Replace a path at particular index in the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__replace_path(const char *path, unsigned int index) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + HDassert(index < H5PL_path_capacity_g); + + /* Insert the path at the requested index */ + if (H5PL__replace_at(path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__insert_path + * + * Purpose: Insert a path at particular index in the table, moving + * any existing paths back to make space. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__insert_path(const char *path, unsigned int index) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + HDassert(index < H5PL_path_capacity_g); + + /* Insert the path at the requested index */ + if (H5PL__insert_at(path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__insert_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__remove_path + * + * Purpose: Remove a path at particular index in the table, freeing + * the path string and moving the paths down to close the gap. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__remove_path(unsigned int index) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(index < H5PL_path_capacity_g); + + /* Check if the path at that index is set */ + if (!H5PL_paths_g[index]) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "search path at index %u is NULL", index) + + /* Delete the path */ + H5PL_num_paths_g--; + H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]); + + /* Shift the paths down to close the gap */ + for (u = index; u < H5PL_num_paths_g; u++) + H5PL_paths_g[u] = H5PL_paths_g[u+1]; + + /* Set the (former) last path to NULL */ + H5PL_paths_g[H5PL_num_paths_g] = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__remove_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_path + * + * Purpose: Get a pointer to a path at particular index in the table. + * + * Return: Success: A pointer to a path string stored in the table + * Failure: NULL + * + *------------------------------------------------------------------------- + */ +const char * +H5PL__get_path(unsigned int index) +{ + char *ret_value = NULL; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Get the path at the requested index */ + if (index >= H5PL_num_paths_g) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "path index %u is out of range in table", index) + + return H5PL_paths_g[index]; +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_path_table + * + * Purpose: Attempts to find a matching plugin in the file system + * using the paths stored in the path table. + *. + * The 'found' parameter will be set appropriately. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(plugin_info); + + /* Initialize output parameters */ + *found = FALSE; + *plugin_info = NULL; + + /* Loop over the paths in the table, checking for an appropriate plugin */ + for (u = 0; u < H5PL_num_paths_g; u++) { + + /* Search for the plugin in this path */ + if (H5PL__find_plugin_in_path(search_params, found, H5PL_paths_g[u], plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in path %s encountered an error", H5PL_paths_g[u]) + + /* Break out if found */ + if (*found) { + if (!plugin_info) + HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "plugin info should not be NULL") + break; + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_path + * + * Purpose: Given a path, this function opens the directory and envokes + * another function to go through all files to find the right + * plugin library. Two function definitions are for Unix and + * Windows. + * + * The found parameter will be set to TRUE and the info + * parameter will be filled in on success. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +#ifndef H5_HAVE_WIN32_API +static herr_t +H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info) +{ + char *path = NULL; + DIR *dirp = NULL; /* Directory stream */ + struct dirent *dp = NULL; /* Directory entry */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(dir); + HDassert(plugin_info); + + /* Initialize the found parameter */ + *found = FALSE; + + /* Open the directory */ + if (!(dirp = HDopendir(dir))) + HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir) + + /* Iterate through all entries in the directory */ + while (NULL != (dp = HDreaddir(dirp))) { + + /* The library we are looking for should be called libxxx.so... on Unix + * or libxxx.xxx.dylib on Mac. + */ +#ifndef __CYGWIN__ + if (!HDstrncmp(dp->d_name, "lib", (size_t)3) && + (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) { +#else + if (!HDstrncmp(dp->d_name, "cyg", (size_t)3) && + HDstrstr(dp->d_name, ".dll") ) { +#endif + + h5_stat_t my_stat; + size_t len; + + /* Allocate & initialize the path name */ + len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(dp->d_name) + 1 /*\0*/; + + if (NULL == (path = (char *)H5MM_calloc(len))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + + HDsnprintf(path, len, "%s/%s", dir, dp->d_name); + + /* Get info for directory entry */ + if (HDstat(path, &my_stat) == -1) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file %s -- error was: %s", path, HDstrerror(errno)) + + /* If it is a directory, skip it */ + if (S_ISDIR(my_stat.st_mode)) + continue; + + /* attempt to open the dynamic library as a filter library */ + if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") + if (*found) + HGOTO_DONE(SUCCEED) + + path = (char *)H5MM_xfree(path); + } /* end if */ + } /* end while */ + +done: + if (dirp) + if (HDclosedir(dirp) < 0) + HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno)) + + path = (char *)H5MM_xfree(path); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path() */ +#else /* H5_HAVE_WIN32_API */ +static herr_t +H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info) +{ + WIN32_FIND_DATAA fdFile; + HANDLE hFind = INVALID_HANDLE_VALUE; + char *path = NULL; + char service[2048]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(dir); + HDassert(plugin_info); + + /* Initialize the found parameter */ + *found = FALSE; + + /* Specify a file mask. *.* = We want everything! */ + HDsprintf(service, "%s\\*.dll", dir); + if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) + HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory") + + /* Loop over all the files */ + do { + /* Ignore '.' and '..' */ + if (HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) { + + /* XXX: Probably just continue here and move the code below over one tab */ + + size_t len; + + /* Allocate & initialize the path name */ + len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(fdFile.cFileName) + 1; + + if (NULL == (path = (char *)H5MM_calloc(len))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + + HDsnprintf(path, len, "%s\\%s", dir, fdFile.cFileName); + + /* Ignore directories */ + if (fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + + /* attempt to open the dynamic library as a filter library */ + if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") + if (*found) + HGOTO_DONE(SUCCEED) + + path = (char *)H5MM_xfree(path); + } + } while (FindNextFileA(hFind, &fdFile)); + +done: + if (hFind != INVALID_HANDLE_VALUE) + FindClose(hFind); + if (path) + path = (char *)H5MM_xfree(path); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path() */ +#endif /* H5_HAVE_WIN32_API */ + diff --git a/src/H5PLpkg.h b/src/H5PLpkg.h index e356893..0d1c271 100644 --- a/src/H5PLpkg.h +++ b/src/H5PLpkg.h @@ -11,6 +11,12 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +/* + * Purpose: This file contains declarations which are visible only within + * the H5PL package. Source files outside the H5PL package should + * include H5PLprivate.h instead. + */ + #if !(defined H5PL_FRIEND || defined H5PL_MODULE) #error "Do not include this file outside the H5PL package!" #endif @@ -27,13 +33,92 @@ /* Package Private Macros */ /**************************/ -#define H5PL_MAX_PATH_NUM 16 +/* Whether to pre-load pathnames for plugin libraries */ +#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR + + +/****************************/ +/* Macros for supporting */ +/* both Windows and POSIX */ +/****************************/ + +/*******************/ +/* Windows support */ +/*******************/ +/* + * SPECIAL WINDOWS NOTE + * + * Some of the Win32 API functions expand to fooA or fooW depending on + * whether UNICODE or _UNICODE are defined. You MUST explicitly use + * the A version of the functions to force char * behavior until we + * work out a scheme for proper Windows Unicode support. + * + * If you do not do this, people will be unable to incorporate our + * source code into their own CMake builds if they define UNICODE. + */ +#ifdef H5_HAVE_WIN32_API + + /* The path separator on this platform */ +# define H5PL_PATH_SEPARATOR ";" + + /* Handle for dynamic library */ +# define H5PL_HANDLE HINSTANCE + + /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ +# define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) + + /* Get the address of a symbol in dynamic library */ +# define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N) + + /* Close dynamic library */ +# define H5PL_CLOSE_LIB(H) FreeLibrary(H) + + /* Clear error - nothing to do */ +# define H5PL_CLR_ERROR + + /* maximum size for expanding env vars */ +# define H5PL_EXPAND_BUFFER_SIZE 32767 + + typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); + +#else /* H5_HAVE_WIN32_API */ + + /*****************/ + /* POSIX support */ + /*****************/ + + /* The path separator on this platform */ +# define H5PL_PATH_SEPARATOR ":" + + /* Handle for dynamic library */ +# define H5PL_HANDLE void * + + /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ +# define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY) + + /* Get the address of a symbol in dynamic library */ +# define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N) + + /* Close dynamic library */ +# define H5PL_CLOSE_LIB(H) dlclose(H) + + /* Clear error */ +# define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror()) + + typedef const void *(*H5PL_get_plugin_info_t)(void); +#endif /* H5_HAVE_WIN32_API */ /****************************/ /* Package Private Typedefs */ /****************************/ +/* Data used to search for plugins */ +typedef struct H5PL_search_params_t { + H5PL_type_t type; + int id; +} H5PL_search_params_t; + /*****************************/ /* Package Private Variables */ @@ -44,5 +129,31 @@ /* Package Private Prototypes */ /******************************/ +/* Accessors to global variables and flags */ +H5_DLL herr_t H5PL__get_plugin_control_mask(unsigned int *mask /*out*/); +H5_DLL herr_t H5PL__set_plugin_control_mask(unsigned int mask); + +/* Plugin search and manipulation */ +H5_DLL herr_t H5PL__open(const char *libname, H5PL_type_t type, int id, hbool_t *success /*out*/, const void **plugin_info /*out*/); +H5_DLL herr_t H5PL__close(H5PL_HANDLE handle); + +/* Plugin cache calls */ +H5_DLL herr_t H5PL__create_plugin_cache(void); +H5_DLL herr_t H5PL__close_plugin_cache(hbool_t *already_closed /*out*/); +H5_DLL herr_t H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle); +H5_DLL herr_t H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/); + +/* Plugin search path calls */ +H5_DLL herr_t H5PL__create_path_table(void); +H5_DLL herr_t H5PL__close_path_table(void); +H5_DLL unsigned H5PL__get_num_paths(void); +H5_DLL herr_t H5PL__append_path(const char *path); +H5_DLL herr_t H5PL__prepend_path(const char *path); +H5_DLL herr_t H5PL__replace_path(const char *path, unsigned int index); +H5_DLL herr_t H5PL__insert_path(const char *path, unsigned int index); +H5_DLL herr_t H5PL__remove_path(unsigned int index); +H5_DLL const char *H5PL__get_path(unsigned int index); +H5_DLL herr_t H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/); + #endif /* _H5PLpkg_H */ diff --git a/src/H5PLplugin_cache.c b/src/H5PLplugin_cache.c new file mode 100644 index 0000000..d826ba0 --- /dev/null +++ b/src/H5PLplugin_cache.c @@ -0,0 +1,307 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Code to implement a plugin cache which stores information + * about plugins which have already been loaded. + * + * The plugin cache is implemented as a dynamic, global array which + * will grow as new plugins are added. The capacity of the cache + * never shrinks since plugins stay in memory once loaded. + * + * Note that this functionality has absolutely nothing to do with + * the metadata or chunk caches. + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ +#include "H5Zprivate.h" /* Filter pipeline */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Initial capacity of the plugin cache */ +#define H5PL_INITIAL_CACHE_CAPACITY 16 + +/* The amount to add to the capacity when the cache is full */ +#define H5PL_CACHE_CAPACITY_ADD 16 + + +/******************/ +/* Local Typedefs */ +/******************/ + +/* Type for the list of info for opened plugin libraries */ +typedef struct H5PL_plugin_t { + H5PL_type_t type; /* Plugin type */ + int id; /* ID for the plugin */ + H5PL_HANDLE handle; /* Plugin handle */ +} H5PL_plugin_t; + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5PL__expand_cache(void); + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Cache for storing opened plugin libraries */ +static H5PL_plugin_t *H5PL_cache_g = NULL; + +/* The number of stored plugins */ +static unsigned int H5PL_num_plugins_g = 0; + +/* The capacity of the plugin cache */ +static unsigned int H5PL_cache_capacity_g = 0; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__create_plugin_cache + * + * Purpose: Create the cache that will store plugins that have already + * been loaded. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__create_plugin_cache(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Allocate memory for the plugin cache */ + H5PL_num_plugins_g = 0; + + H5PL_cache_capacity_g = H5PL_INITIAL_CACHE_CAPACITY; + + if (NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_calloc((size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for plugin cache") + +done: + /* Try to clean up on errors */ + if (FAIL == ret_value) { + if (H5PL_cache_g) + H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g); + H5PL_cache_capacity_g = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__create_plugin_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__close_plugin_cache + * + * Purpose: Close the cache of plugins that have already been loaded, + * closing all the plugins contained inside. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close_plugin_cache(hbool_t *already_closed /*out*/) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + /* Close opened dynamic libraries */ + if (H5PL_cache_g) { + + /* Close any cached plugins */ + for (u = 0; u < H5PL_num_plugins_g; u++) + H5PL__close((H5PL_cache_g[u]).handle); + + /* Free the cache array */ + H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g); + H5PL_num_plugins_g = 0; + H5PL_cache_capacity_g = 0; + + /* Note that actually closed the table (needed by package close call) */ + *already_closed = FALSE; + } + else + *already_closed = TRUE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__close_plugin_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__expand_cache + * + * Purpose: Expand the plugin cache when it's full. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__expand_cache(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Update the capacity */ + H5PL_cache_capacity_g += H5PL_CACHE_CAPACITY_ADD; + + /* Resize the array */ + if(NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_realloc(H5PL_cache_g, (size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for plugin cache failed") + + /* Initialize the new memory */ + HDmemset(H5PL_cache_g + H5PL_num_plugins_g, 0, (size_t)H5PL_CACHE_CAPACITY_ADD * sizeof(H5PL_plugin_t)); + +done: + /* Set the cache capacity back if there were problems */ + if (FAIL == ret_value) + H5PL_cache_capacity_g -= H5PL_CACHE_CAPACITY_ADD; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__expand_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__add_plugin + * + * Purpose: Add a plugin to the plugin cached. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Expand the cache if it is too small */ + if (H5PL_num_plugins_g >= H5PL_cache_capacity_g) + if (H5PL__expand_cache() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand plugin cache") + + /* Store the plugin info and bump the # of plugins */ + H5PL_cache_g[H5PL_num_plugins_g].type = type; + H5PL_cache_g[H5PL_num_plugins_g].id = id; + H5PL_cache_g[H5PL_num_plugins_g].handle = handle; + + H5PL_num_plugins_g++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__add_plugin() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_cache + * + * Purpose: Attempts to find a matching plugin from the cache. + * + * The 'found' parameter will be set appropriately. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +/* See the other use of H5PL_GET_LIB_FUNC() for an explanation + * for why we disable -Wpedantic here. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +herr_t +H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(plugin_info); + + /* Initialize output parameters */ + *found = FALSE; + *plugin_info = NULL; + + /* Loop over all the plugins, looking for one that matches */ + for (u = 0; u < H5PL_num_plugins_g; u++) { + + /* If the plugin type (filter, etc.) and ID match, query the plugin for its info */ + if ((search_params->type == (H5PL_cache_g[u]).type) && (search_params->id == (H5PL_cache_g[u]).id)) { + + H5PL_get_plugin_info_t get_plugin_info_function; + const H5Z_class2_t *filter_info; + + /* Get the "get plugin info" function from the plugin. */ + if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_cache_g[u]).handle, "H5PLget_plugin_info"))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") + + /* Call the "get plugin info" function */ + if (NULL == (filter_info = (const H5Z_class2_t *)(*get_plugin_info_function)())) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info") + + /* Set output parameters */ + *found = TRUE; + *plugin_info = filter_info; + + /* No need to continue processing */ + break; + + } /* end if */ + + } /* end for */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_cache() */ +#pragma GCC diagnostic pop + diff --git a/src/H5PLpublic.h b/src/H5PLpublic.h index 9ce1fca..3b36ccd 100644 --- a/src/H5PLpublic.h +++ b/src/H5PLpublic.h @@ -10,8 +10,8 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Raymond Lu <songyulu@hdfgroup.org> - * 13 February 2013 +/* + * This file contains public declarations for the H5PL module. */ #ifndef _H5PLpublic_H @@ -24,31 +24,35 @@ /* Public Typedefs */ /*******************/ +/* Special string to indicate no plugin loading. + */ +#define H5PL_NO_PLUGIN "::" + /* Plugin type used by the plugin library */ typedef enum H5PL_type_t { - H5PL_TYPE_ERROR = -1, /*error */ - H5PL_TYPE_FILTER = 0, /*filter */ - H5PL_TYPE_NONE = 1 /*this must be last! */ + H5PL_TYPE_ERROR = -1, /* Error */ + H5PL_TYPE_FILTER = 0, /* Filter */ + H5PL_TYPE_NONE = 1 /* This must be last! */ } H5PL_type_t; /* Common dynamic plugin type flags used by the set/get_loading_state functions */ -#define H5PL_FILTER_PLUGIN 0x0001 -#define H5PL_ALL_PLUGIN 0xFFFF +#define H5PL_FILTER_PLUGIN 0x0001 +#define H5PL_ALL_PLUGIN 0xFFFF #ifdef __cplusplus extern "C" { #endif /* plugin state */ -H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_type); -H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_type/*out*/); -H5_DLL herr_t H5PLappend(const char *plugin_path); -H5_DLL herr_t H5PLprepend(const char *plugin_path); -H5_DLL herr_t H5PLreplace(const char *plugin_path, unsigned int index); -H5_DLL herr_t H5PLinsert(const char *plugin_path, unsigned int index); +H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_control_mask); +H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_control_mask /*out*/); +H5_DLL herr_t H5PLappend(const char *search_path); +H5_DLL herr_t H5PLprepend(const char *search_path); +H5_DLL herr_t H5PLreplace(const char *search_path, unsigned int index); +H5_DLL herr_t H5PLinsert(const char *search_path, unsigned int index); H5_DLL herr_t H5PLremove(unsigned int index); -H5_DLL ssize_t H5PLget(unsigned int index, char *pathname/*out*/, size_t size); -H5_DLL herr_t H5PLsize(unsigned int *listsize/*out*/); +H5_DLL ssize_t H5PLget(unsigned int index, char *path_buf /*out*/, size_t buf_size); +H5_DLL herr_t H5PLsize(unsigned int *num_paths /*out*/); #ifdef __cplusplus } diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 55b3877..854b1ef 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -166,7 +166,7 @@ typedef enum H5D_mpio_no_collective_cause_t { H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED = 0x08, H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES = 0x10, H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET = 0x20, - H5D_MPIO_FILTERS = 0x40 + H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x40 } H5D_mpio_no_collective_cause_t; /********************/ @@ -105,7 +105,7 @@ H5R__init_package(void) /* Initialize the atom group for the file IDs */ if(H5I_register_type(H5I_REFERENCE_CLS) < 0) - HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINIT, FAIL, "unable to initialize interface") + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINIT, FAIL, "unable to initialize interface") /* Mark "top" of interface as initialized, too */ H5R_top_package_initialize_s = TRUE; @@ -141,10 +141,10 @@ H5R_top_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR if(H5R_top_package_initialize_s) { - if(H5I_nmembers(H5I_REFERENCE) > 0) { - (void)H5I_clear_type(H5I_REFERENCE, FALSE, FALSE); + if(H5I_nmembers(H5I_REFERENCE) > 0) { + (void)H5I_clear_type(H5I_REFERENCE, FALSE, FALSE); n++; /*H5I*/ - } /* end if */ + } /* end if */ /* Mark closed */ if(0 == n) @@ -741,7 +741,7 @@ H5Rget_region(hid_t id, H5R_type_t ref_type, const void *ref) HGOTO_ERROR(H5E_REFERENCE, H5E_CANTCREATE, FAIL, "unable to create dataspace") /* Atomize */ - if((ret_value = H5I_register (H5I_DATASPACE, space, TRUE)) < 0) + if((ret_value = H5I_register(H5I_DATASPACE, space, TRUE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace atom") done: diff --git a/src/H5err.txt b/src/H5err.txt index 3f5801f..d771956 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -243,6 +243,8 @@ MINOR, LINK, H5E_CANTSORT, Can't sort objects MINOR, MPI, H5E_MPI, Some MPI function failed MINOR, MPI, H5E_MPIERRSTR, MPI Error String MINOR, MPI, H5E_CANTRECV, Can't receive data +MINOR, MPI, H5E_CANTGATHER, Can't gather data +MINOR, MPI, H5E_NO_INDEPENDENT, Can't perform independent IO # Heap errors MINOR, HEAP, H5E_CANTRESTORE, Can't restore condition diff --git a/src/H5private.h b/src/H5private.h index c588154..ca4ebcf 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -495,6 +495,15 @@ # define H5_POSIX_MAX_IO_BYTES SSIZET_MAX #endif +/* POSIX I/O mode used as the third parameter to open/_open + * when creating a new file (O_CREAT is set). + */ +#if defined(H5_HAVE_WIN32_API) +# define H5_POSIX_CREATE_MODE_RW (_S_IREAD | _S_IWRITE) +#else +# define H5_POSIX_CREATE_MODE_RW 0666 +#endif + /* * A macro to portably increment enumerated types. */ @@ -1117,11 +1126,7 @@ typedef off_t h5_stat_size_t; #define HDnanosleep(N, O) nanosleep(N, O) #endif /* HDnanosleep */ #ifndef HDopen - #ifdef _O_BINARY - #define HDopen(S,F,M) open(S,F|_O_BINARY,M) - #else - #define HDopen(S,F,M) open(S,F,M) - #endif + #define HDopen(F,...) open(F,__VA_ARGS__) #endif /* HDopen */ #ifndef HDopendir #define HDopendir(S) opendir(S) diff --git a/src/H5system.c b/src/H5system.c index 7e25540..a1cdf19 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -1237,3 +1237,49 @@ H5_get_time(void) } /* end H5_get_time() */ +#ifdef H5_HAVE_WIN32_API + +#define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767 + + +/*------------------------------------------------------------------------- + * Function: H5_expand_windows_env_vars() + * + * Purpose: Replaces windows environment variables of the form %foo% + * with user-specific values. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_expand_windows_env_vars(char **env_var) +{ + long n_chars = 0; + char *temp_buf = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Allocate buffer for expanded environment variable string */ + if (NULL == (temp_buf = (char *)H5MM_calloc((size_t)H5_WIN32_ENV_VAR_BUFFER_SIZE))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") + + /* Expand the environment variable string */ + if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") + + if (0 == n_chars) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") + + *env_var = (char *)H5MM_xfree(*env_var); + *env_var = temp_buf; + +done: + if (FAIL == ret_value && temp_buf) + temp_buf = (char *)H5MM_xfree(temp_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_expand_windows_env_vars() */ +#endif /* H5_HAVE_WIN32_API */ + diff --git a/src/H5trace.c b/src/H5trace.c index 9fb8a72..930002f 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -621,10 +621,6 @@ H5_trace(const double *returning, const char *func, const char *type, ...) fprintf(out, "%sH5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET", flag_already_displayed ? " | " : ""); flag_already_displayed = TRUE; } /* end if */ - if(nocol_cause_mode & H5D_MPIO_FILTERS) { - fprintf(out, "%sH5D_MPIO_FILTERS", flag_already_displayed ? " | " : ""); - flag_already_displayed = TRUE; - } /* end if */ /* Display '<none>' if there's no flags set */ if(!flag_already_displayed) diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 0149faa..4522228 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -48,9 +48,13 @@ typedef __int64 h5_stat_size_t; #define HDnanosleep(N, O) Wnanosleep(N, O) #define HDoff_t __int64 /* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL - * transformations when performing I/O. + * transformations when performing I/O. Note that this will + * produce Unix-style text files, though. + * + * Also note that the variadic macro is using a VC++ extension + * where the comma is dropped if nothing is passed to the ellipsis. */ -#define HDopen(S,F,M) _open(S,F|_O_BINARY,M) +#define HDopen(S,F,...) _open(S, F | _O_BINARY, __VA_ARGS__) #define HDread(F,M,Z) _read(F,M,Z) #define HDrmdir(S) _rmdir(S) #define HDsetvbuf(F,S,M,Z) setvbuf(F,S,M,(Z>1?Z:2)) @@ -115,6 +119,7 @@ extern "C" { H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...); H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap); H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem); + H5_DLL herr_t H5_expand_windows_env_vars(char **env_var); /* Round functions only needed for VS2012 and earlier. * They are always built to ensure they don't go stale and diff --git a/src/Makefile.am b/src/Makefile.am index 0b664a7..9a64717 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -97,7 +97,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Pgcpl.c H5Pint.c \ H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \ H5PB.c \ - H5PL.c \ + H5PL.c H5PLint.c H5PLpath.c H5PLplugin_cache.c \ H5R.c H5Rdeprec.c \ H5UC.c \ H5RS.c \ |