summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorRichard Warren <Richard.Warren@hdfgroup.org>2019-09-23 17:28:51 (GMT)
committerRichard Warren <Richard.Warren@hdfgroup.org>2019-09-23 17:28:51 (GMT)
commit4ff30ecef467d50df7b27296126f763cb7d97038 (patch)
tree15f31db19d96bbe7e7da713d6d7a4abc20be2578 /src
parent1faa2466ccb182e92471754d21b3fde408287f2a (diff)
downloadhdf5-4ff30ecef467d50df7b27296126f763cb7d97038.zip
hdf5-4ff30ecef467d50df7b27296126f763cb7d97038.tar.gz
hdf5-4ff30ecef467d50df7b27296126f763cb7d97038.tar.bz2
Making a testing branch of Q&I functionality available for testing
Diffstat (limited to 'src')
-rw-r--r--src/H5Dchunk.c10
-rw-r--r--src/H5Dprivate.h2
-rw-r--r--src/H5FL.c9
-rw-r--r--src/H5Q.c5
-rw-r--r--src/H5Shyper.c8
-rw-r--r--src/H5Sselect.c2
-rw-r--r--src/H5X.c31
-rw-r--r--src/H5Xfastbit.c1229
-rw-r--r--src/H5Xmeta_db.c54
9 files changed, 223 insertions, 1127 deletions
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index 7e73a4a..0ab63f8 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -700,6 +700,12 @@ H5D__chunk_is_space_alloc(const H5O_storage_t *storage)
} /* end H5D__chunk_is_space_alloc() */
+static int io_init_count = 0;
+void H5D_reset_io_init_count(void)
+{
+ io_init_count = 0;
+}
+
/*-------------------------------------------------------------------------
* Function: H5D__chunk_io_init
*
@@ -957,8 +963,10 @@ H5D__chunk_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_inf
chunk_info = (H5D_chunk_info_t *)H5SL_item(curr_node);
HDassert(chunk_info);
+ io_init_count++; /* Debuginfo */
+
/* Clean hyperslab span's "scratch" information */
- if(H5S_hyper_reset_scratch(chunk_info->mspace) < 0)
+ if(chunk_info->mspace && (H5S_hyper_reset_scratch(chunk_info->mspace) < 0))
HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset span scratch info")
/* Get the next chunk node in the skip list */
diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index 34fa1fb..4930aeb 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -218,5 +218,7 @@ H5_DLL herr_t H5D_btree_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stre
H5_DLL herr_t H5D_write(H5D_t *dset, hbool_t direct_write, hid_t mem_type_id,
const H5S_t *mem_space, const H5S_t *file_space, hid_t dxpl_id, const void *buf);
+H5_DLL void H5D_reset_io_init_count(void);
+
#endif /* _H5Dprivate_H */
diff --git a/src/H5FL.c b/src/H5FL.c
index fb355ad..88a4b09 100644
--- a/src/H5FL.c
+++ b/src/H5FL.c
@@ -528,6 +528,8 @@ H5FL__reg_gc_list(H5FL_reg_head_t *head)
{
H5FL_reg_node_t *free_list; /* Pointer to nodes in free list being garbage collected */
size_t total_mem; /* Total memory used on list */
+ void *freed[8192];
+ int k, total = 0;
FUNC_ENTER_STATIC_NOERR
@@ -536,6 +538,7 @@ H5FL__reg_gc_list(H5FL_reg_head_t *head)
/* For each free list being garbage collected, walk through the nodes and free them */
free_list = head->list;
+
while(free_list != NULL) {
void *tmp; /* Temporary node pointer */
@@ -543,9 +546,11 @@ H5FL__reg_gc_list(H5FL_reg_head_t *head)
/* Decrement the count of nodes allocated and free the node */
head->allocated--;
-
+ for(k=0; k<total; k++)
+ if (freed[k] == free_list) break;
+ if (k == total)
H5MM_free(free_list);
-
+ freed[total++] = free_list;
free_list = (H5FL_reg_node_t *)tmp;
} /* end while */
diff --git a/src/H5Q.c b/src/H5Q.c
index 39c5545..fda4006 100644
--- a/src/H5Q.c
+++ b/src/H5Q.c
@@ -377,11 +377,6 @@ H5Q__init_package(void)
FUNC_ENTER_PACKAGE
-#ifdef H5_HAVE_PARALLEL
- if (FAIL == H5Xinitialize_parallel_query())
- printf("Unable to initialize parallel queries\n");
-#endif
-
/* Initialize the atom group for the QUERY IDs */
if (FAIL == H5I_register_type(H5I_QUERY_CLS))
HGOTO_ERROR(H5E_QUERY, H5E_CANTINIT, FAIL, "unable to initialize interface");
diff --git a/src/H5Shyper.c b/src/H5Shyper.c
index 774e79d..dea9e21 100644
--- a/src/H5Shyper.c
+++ b/src/H5Shyper.c
@@ -8029,6 +8029,13 @@ H5S_combine_select(H5S_t *space1, H5S_seloper_t op, H5S_t *space2)
HDassert(space2);
HDassert(op >= H5S_SELECT_OR && op <= H5S_SELECT_NOTA);
+ if (space1->select.type->type == H5S_SEL_NONE) {
+ if (op == H5S_SELECT_AND)
+ ret_value = new_space = H5S_copy(space1, FALSE, TRUE);
+ else if (op == H5S_SELECT_OR)
+ ret_value = new_space = H5S_copy(space2, FALSE, TRUE);
+ goto done;
+ }
/* Check that if space1 selections has span trees */
if(NULL == space1->select.sel_info.hslab->span_lst)
if(H5S_hyper_generate_spans(space1) < 0)
@@ -8041,7 +8048,6 @@ H5S_combine_select(H5S_t *space1, H5S_seloper_t op, H5S_t *space2)
ret_value = new_space = H5S_copy(space1, FALSE, TRUE);
goto done;
}
-
if(NULL == space2->select.sel_info.hslab->span_lst) {
hsize_t tmp_start[H5S_MAX_RANK];
diff --git a/src/H5Sselect.c b/src/H5Sselect.c
index b4af9d4..0bbb04a 100644
--- a/src/H5Sselect.c
+++ b/src/H5Sselect.c
@@ -1057,7 +1057,7 @@ H5S_select_iter_init(H5S_sel_iter_t *sel_iter, const H5S_t *space, size_t elmt_s
/* Call initialization routine for selection type */
ret_value = (*space->select.type->iter_init)(sel_iter, space);
- HDassert(sel_iter->type);
+ HDassert(sel_iter->type != NULL);
FUNC_LEAVE_NOAPI(ret_value)
} /* H5S_select_iter_init() */
diff --git a/src/H5X.c b/src/H5X.c
index 2a6e927..5624707 100644
--- a/src/H5X.c
+++ b/src/H5X.c
@@ -138,6 +138,15 @@ H5X__init_package(void)
#ifdef H5_HAVE_FASTBIT
if (H5X_register(H5X_FASTBIT) < 0)
HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register FastBit index plugin");
+#ifdef H5_HAVE_PARALLEL
+ { /* Enable parallel queries if MPI already initialized and we are using the fastbit plugin */
+ int mpi_available;
+ if (MPI_SUCCESS == MPI_Initialized(&mpi_available))
+ if (mpi_available) {
+ H5Xinitialize_parallel_query();
+ }
+ }
+#endif
#endif
if (H5X_register(H5X_META_DUMMY) < 0)
HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register meta dummy index plugin");
@@ -457,10 +466,15 @@ herr_t
H5Xcreate(hid_t loc_id, unsigned plugin_id, hid_t xcpl_id)
{
herr_t ret_value = SUCCEED; /* Return value */
-
+#ifdef H5_HAVE_PARALLEL
+ double start_t, end_t;
+#endif
FUNC_ENTER_API(FAIL)
H5TRACE3("e", "iIui", loc_id, plugin_id, xcpl_id);
+#ifdef H5_HAVE_PARALLEL
+ start_t = MPI_Wtime();
+#endif
/* Check args */
if (plugin_id > H5X_PLUGIN_MAX)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid plugin identification number");
@@ -470,6 +484,11 @@ H5Xcreate(hid_t loc_id, unsigned plugin_id, hid_t xcpl_id)
if (FAIL == H5X_create(loc_id, plugin_id, xcpl_id))
HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "cannot create index");
+#ifdef H5_HAVE_PARALLEL
+ end_t = MPI_Wtime();
+ printf("Time to create index = %lf Seconds\n", (end_t - start_t));
+#endif
+
done:
FUNC_LEAVE_API(ret_value)
} /* end H5Xcreate() */
@@ -1091,12 +1110,16 @@ H5Xinitialize_parallel_query()
{
static int initialized = 0;
herr_t ret = SUCCEED;
+#ifdef H5_HAVE_PARALLEL
MPI_Comm comm = MPI_COMM_WORLD;
if (!initialized) {
- gather_topology_info(comm);
- ret = create_group_comms(comm);
- if (ret == SUCCEED) initialized++;
+ initialized = 1;
+ if (MPI_Comm_size(comm, &g_mpi_size) != MPI_SUCCESS)
+ return FAIL;
+ if (g_mpi_size > 1)
+ ret = create_group_comms(comm);
}
+#endif
return ret;
}
diff --git a/src/H5Xfastbit.c b/src/H5Xfastbit.c
index 0eb71a3..14a9858 100644
--- a/src/H5Xfastbit.c
+++ b/src/H5Xfastbit.c
@@ -24,7 +24,7 @@
/* Headers */
/***********/
#include "H5private.h" /* Generic Functions */
-#include "H5Xprivate.h" /* Index */
+#include "H5Xprivate.h" /* Index */
#include "H5Eprivate.h" /* Error handling */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
@@ -51,13 +51,10 @@
/****************/
/* Local Macros */
/****************/
-#define LIMIT_RESULTS_FOR_TESTING
-#define SHOW_TIME
-// #define H5X_FASTBIT_DEBUG
+//#define H5X_FASTBIT_DEBUG
#ifdef H5X_FASTBIT_DEBUG
#define H5X_FASTBIT_DEBUG_LVL 6
-// #define H5X_FASTBIT_DEBUG_LVL 2
#define H5X_FASTBIT_LOG_DEBUG(...) do { \
fprintf(stdout, " # %s(): ", __func__); \
fprintf(stdout, __VA_ARGS__); \
@@ -72,14 +69,6 @@
#define H5X_FASTBIT_MAX_NAME 1024
-struct fastbit_meta_collection {
- /* hsize_t base_offset; // We'll eventually need this too... */
- hsize_t data_length;
- hsize_t nkeys;
- hsize_t noffsets;
- hsize_t nbitmaps;
-};
-
/******************/
/* Local Typedefs */
/******************/
@@ -88,7 +77,6 @@ typedef struct H5X_fastbit_t {
hid_t file_id; /* ID of the indexed dataset file */
hid_t dataset_id; /* ID of the indexed dataset */
-
unsigned dataset_ndims; /* dataset number of dimensions */
hsize_t *dataset_dims; /* dataset dimensions */
hsize_t *dataset_down_dims; /* dataset downed dimensions */
@@ -128,23 +116,11 @@ struct H5X_fastbit_scatter_info {
size_t src_buf_size; /* Remaining number of elements to return */
};
-typedef struct H5X_fastbit_list {
- hid_t dataset_id;
- H5X_fastbit_t *fastbit_info;
- struct H5X_fastbit_list *next;
-} H5X_fastbit_list_t;
-
-
-static H5X_fastbit_list_t *H5X_fastbit_current_datasets = NULL;
-
/********************/
/* Local Prototypes */
/********************/
static H5X_fastbit_t *
-H5X__get_fastbit(hid_t dataset_id);
-
-static H5X_fastbit_t *
H5X__fastbit_init(hid_t dataset_id);
static herr_t
@@ -164,13 +140,6 @@ static herr_t
H5X__fastbit_build_index(H5X_fastbit_t *fastbit);
static herr_t
-H5X__fastbit_define_dataset(H5X_fastbit_t *fastbit, int typeIndex, struct fastbit_meta_collection *gatherInfo,
- hsize_t *array_size, hsize_t *offset, hid_t *space_id, hid_t *memspace_id);
-
-static herr_t
-H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit);
-
-static herr_t
H5X__fastbit_serialize_metadata(H5X_fastbit_t *fastbit, void *buf,
size_t *buf_size);
@@ -178,9 +147,6 @@ static herr_t
H5X__fastbit_deserialize_metadata(H5X_fastbit_t *fastbit, void *buf);
static herr_t
-H5X__fastbit_reconstruct_parallel_index(H5X_fastbit_t *fastbit);
-
-static herr_t
H5X__fastbit_reconstruct_index(H5X_fastbit_t *fastbit);
static double
@@ -262,106 +228,6 @@ const H5X_class_t H5X_FASTBIT[1] = {{
}};
/*-------------------------------------------------------------------------
- * Function: H5X__fastbit_add_to_list
- *
- * Purpose: Creates and then adds a H5X_fastbit_list_t entry
- * to a list of known datasets.
- *-------------------------------------------------------------------------
- */
-
-static H5X_fastbit_list_t *
-H5X__fastbit_add_to_list(H5X_fastbit_t *fastbit_info)
-{
- H5X_fastbit_list_t *ret_value = NULL;
- H5X_fastbit_list_t *new_entry = NULL;
-
- FUNC_ENTER_NOAPI_NOINIT
- HDassert(fastbit_info);
-
- if (NULL == (new_entry = (H5X_fastbit_list_t *)H5MM_malloc(sizeof(H5X_fastbit_list_t))))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, NULL, "can't allocate FastBit list struct");
-
- new_entry->dataset_id = fastbit_info->dataset_id;
- new_entry->fastbit_info = fastbit_info;
- new_entry->next = H5X_fastbit_current_datasets;
- H5X_fastbit_current_datasets = new_entry;
- ret_value = new_entry;
-
- done:
- FUNC_LEAVE_NOAPI(ret_value)
-}
-
-/*-------------------------------------------------------------------------
- * Function: H5X__fastbit_remove_from_list
- *
- * Purpose: Creates and then adds a H5X_fastbit_list_t entry
- * to a list of known datasets.
- *-------------------------------------------------------------------------
- */
-static void H5X__fastbit_remove_from_list(hid_t dataset_id)
-{
- H5X_fastbit_list_t *thisEntry = H5X_fastbit_current_datasets;
- H5X_fastbit_list_t *previous = NULL;
- while (thisEntry && thisEntry->dataset_id != dataset_id) {
- previous = thisEntry;
- thisEntry = thisEntry->next;
- }
- if (thisEntry == H5X_fastbit_current_datasets)
- H5X_fastbit_current_datasets = thisEntry->next;
- if (previous) previous->next = thisEntry->next;
- if (thisEntry) H5MM_free(thisEntry);
-}
-
-/*-------------------------------------------------------------------------
- * Function: H5X__get_fastbit
- *
- * Purpose: Searchs the list of known datasets and returns a
- * H5X_fastbit_t pointer if it contains the requested dataset_id.
- *-------------------------------------------------------------------------
- */
-static H5X_fastbit_t *
-H5X__get_fastbit(hid_t dataset_id)
-{
- H5X_fastbit_list_t *nextEntry = H5X_fastbit_current_datasets;
- while(nextEntry) {
- if (nextEntry->dataset_id == dataset_id) return nextEntry->fastbit_info;
- nextEntry = nextEntry->next;
- }
- return NULL;
-}
-
-
-static void
-dump_as_bytes(char *entry, hsize_t length, void *data)
-{
- static int mpi_rank = -1;
- char filename[80];
- FILE *save_dump;
- unsigned char *show = (char *)data;
- int i, k;
- int rows = (int)(length/10);
- int lastrow = (int)(length % 10);
- if (mpi_rank < 0) mpi_rank = H5Xparallel_rank();
- sprintf(filename, "%s-dump-%d.txt", entry, mpi_rank);
- save_dump = fopen(filename, "a+");
-
- fprintf(save_dump, "-----------\n\n%s length is %lld bytes::\n",entry, length);
- for (i=0; i< rows; i++) {
- fprintf(save_dump,"%4d\t%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n", i, show[0],
- show[1],show[2],show[3],show[4],show[5],show[6],show[7],show[8],show[9]);
- show += 10;
- }
- if (lastrow) {
- fprintf(save_dump,"%4d\t%02x", rows, show[0]);
- for(i=1; i< lastrow; i++)
- fprintf(save_dump," %02x", show[i]);
- fprintf(save_dump,"\n");
- }
- fclose(save_dump);
-}
-
-
-/*-------------------------------------------------------------------------
* Function: H5X__fastbit_init
*
* Purpose: Configure and set up and the FastBit encoder.
@@ -447,10 +313,6 @@ H5X__fastbit_init(hid_t dataset_id)
ret_value = fastbit;
-#ifdef H5_HAVE_PARALLEL
- H5X__fastbit_add_to_list(fastbit);
-#endif
-
done:
if (space_id != FAIL)
H5Sclose(space_id);
@@ -472,6 +334,7 @@ H5X__fastbit_term(H5X_fastbit_t *fastbit)
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
+
if (!fastbit)
HGOTO_DONE(SUCCEED);
@@ -495,9 +358,6 @@ H5X__fastbit_term(H5X_fastbit_t *fastbit)
HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close opaque type");
/* Close anonymous datasets */
- if ((FAIL != fastbit->index_info_group_id) &&
- (FAIL == H5Dclose(fastbit->index_info_group_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close anonymous dataset for index");
if ((FAIL != fastbit->keys_id) &&
(FAIL == H5Dclose(fastbit->keys_id)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close anonymous dataset for index");
@@ -514,10 +374,6 @@ H5X__fastbit_term(H5X_fastbit_t *fastbit)
/* Free FastBit resources */
fastbit_cleanup();
-#ifdef H5_HAVE_PARALLEL
- H5X__fastbit_remove_from_list(fastbit->dataset_id);
-#endif
-
H5MM_free(fastbit);
done:
@@ -612,10 +468,6 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
{
herr_t ret_value = SUCCEED; /* Return value */
hid_t type_id = FAIL, space_id = FAIL;
- hid_t mem_space = H5S_ALL;
- hid_t file_space = H5P_DEFAULT;
- hid_t dxpl_id = H5P_DEFAULT;
-
size_t nelmts, elmt_size;
void *data = NULL;
size_t data_size;
@@ -623,80 +475,25 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
FUNC_ENTER_NOAPI_NOINIT
+ /* Get space info */
if (FAIL == (type_id = H5Dget_type(dataset_id)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get type from dataset");
- if (0 == (elmt_size = H5Tget_size(type_id)))
- HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "invalid size of element");
if (FAIL == (space_id = H5Dget_space(dataset_id)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
-
-#ifdef H5_HAVE_PARALLEL
- if (H5Xparallel_queries_enabled() > 0) {
- int i, ds_ndims = 0;
- hsize_t *start = NULL, *count = NULL, *block = NULL, *stride = NULL;
- H5X_fastbit_t *fastbit = H5X__get_fastbit(dataset_id);
- if (fastbit->filespace_id < 0) fastbit->filespace_id = space_id;
- ds_ndims = H5Xslab_set(fastbit->filespace_id, &start, &count, &stride, &block);
- file_space = fastbit->filespace_id;
-
- if (H5Sselect_hyperslab(fastbit->filespace_id, H5S_SELECT_SET, start, stride, count, block) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Sselect_hyperslab returned an error");
-
- if ((mem_space = H5Screate_simple (ds_ndims, block, NULL)) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
-
- fastbit->memspace_id = mem_space;
- fastbit->filespace_offset = start; /* Save the rank specific file offset */
- nelmts = block[0];
- for (i=1; i< ds_ndims; i++)
- nelmts *= block[i];
-
- data_size = nelmts * elmt_size;
- // printf("data_size: %llu = %llu x %llu\n", data_size, nelmts, elmt_size);
-
- if (NULL == (data = H5MM_malloc(data_size)))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
-
- // printf("H5X__fastbit_read_data: allocated %llu bytes, data=%p\n", data_size, data);
-
- /* The variable 'start' is the original pointer allocated within H5Xslab_set.
- * The additional variables: count, block, and stride are all just offsets
- * from this pointer. As a result, we only need to free the start pointer.
- */
- free(start);
-
- if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "unable to create xfer property");
- if ((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, FAIL, "unable to set MPIO_COLLECTIVE property");
- }
- else {
- /* Get space info */
- if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-
- /* Allocate buffer to hold data */
- data_size = nelmts * elmt_size;
- if (NULL == (data = H5MM_malloc(data_size)))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
- }
-
-#else
if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+ if (0 == (elmt_size = H5Tget_size(type_id)))
+ HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "invalid size of element");
/* Allocate buffer to hold data */
data_size = nelmts * elmt_size;
if (NULL == (data = H5MM_malloc(data_size)))
HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
-#endif
+
/* Read data from dataset */
- if (FAIL == H5Dread(dataset_id, type_id, mem_space, file_space, dxpl_id, data))
+ if (FAIL == H5Dread(dataset_id, type_id, H5S_ALL, space_id, H5P_DEFAULT, data))
HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
- if (dxpl_id != H5P_DEFAULT)
- H5Pclose(dxpl_id);
-
/* Convert type to FastBit type */
if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
@@ -710,17 +507,12 @@ H5X__fastbit_read_data(hid_t dataset_id, const char *column_name, void **buf,
*buf_size = data_size;
done:
- /* NOTE: for parallel indexing, we've made a copy of of the mem_space selection
- * Should we create an actual copy and delete the version we created here?
- */
if (type_id != FAIL)
H5Tclose(type_id);
if (space_id != FAIL)
H5Sclose(space_id);
- if (ret_value == FAIL) {
+ if (ret_value == FAIL)
H5MM_free(data);
- *buf = NULL;
- }
FUNC_LEAVE_NOAPI(ret_value)
} /* H5X__fastbit_read_data() */
@@ -736,18 +528,10 @@ done:
static herr_t
H5X__fastbit_build_index(H5X_fastbit_t *fastbit)
{
- static int sequence_id = 0;
- char log_entry[256];
-
hid_t key_space_id = FAIL, offset_space_id = FAIL, bitmap_space_id = FAIL;
hsize_t key_array_size, offset_array_size, bitmap_array_size;
herr_t ret_value = SUCCEED; /* Return value */
-#ifdef H5_HAVE_PARALLEL
- if (H5Xparallel_queries_enabled() > 0)
- return H5X__fastbit_build_parallel_index(fastbit);
-#endif
-
FUNC_ENTER_NOAPI_NOINIT
H5X_FASTBIT_LOG_DEBUG("Calling FastBit build index");
@@ -830,24 +614,6 @@ H5X__fastbit_build_index(H5X_fastbit_t *fastbit)
H5S_ALL, H5P_DEFAULT, fastbit->bitmaps))
HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't write index metadata");
- if (sequence_id == -1) { /* Disabled for now */
- key_array_size = fastbit->nkeys * sizeof(double);
- offset_array_size = fastbit->noffsets * sizeof(int64_t);
- bitmap_array_size = fastbit->nbitmaps * sizeof(uint32_t);
-
- sprintf(log_entry, "seq-keys-save-%s", fastbit->column_name);
- dump_as_bytes(log_entry, key_array_size, fastbit->keys);
-
- sprintf(log_entry, "seq-offsets-save-%s", fastbit->column_name);
- dump_as_bytes(log_entry, offset_array_size, fastbit->offsets);
-
- sprintf(log_entry, "seq-bitmaps-save-%s", fastbit->column_name);
- dump_as_bytes(log_entry, bitmap_array_size, fastbit->bitmaps);
-
- sequence_id++;
- }
-
-
done:
if (key_space_id != FAIL)
H5Sclose(key_space_id);
@@ -866,380 +632,6 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5X__fastbit_build_index */
-
-#ifdef H5_HAVE_PARALLEL
-#define KEYS_ 0
-#define OFFSETS_ 1
-#define BITMAPS_ 2
-#define INFO_ 3
-#define BADINDEX_ 4
-
-static herr_t
-H5X__fastbit_define_dataset(H5X_fastbit_t *fastbit, int typeIndex, struct fastbit_meta_collection *gatherInfo,
- hsize_t *array_size, hsize_t *offset, hid_t *space_id, hid_t *memspace_id)
-{
- herr_t ret_value = SUCCEED; /* Return value */
- hid_t _space_id = FAIL, _memspace_id = H5S_ALL;;
- int mpi_rank = H5Xparallel_rank();
- hsize_t count;
-
- FUNC_ENTER_NOAPI_NOINIT
-
- if ((typeIndex < 0) || (typeIndex >= BADINDEX_))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "Bad data selection index");
- if (FAIL == (_space_id = H5Screate_simple(1, array_size, NULL)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (file) dataspace");
-
- if (typeIndex == KEYS_) {
- count = (hsize_t)gatherInfo[mpi_rank].nkeys;
- if (FAIL == (_memspace_id = H5Screate_simple(1, &count, NULL)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
- if (H5Sselect_all(_memspace_id) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
- if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, &count, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if (FAIL == (fastbit->keys_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
- _space_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- /* Increment refcount so that anonymous dataset is persistent */
- if (FAIL == H5Oincr_refcount(fastbit->keys_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
- }
-
- else if (typeIndex == OFFSETS_) {
- count = (hsize_t)gatherInfo[mpi_rank].noffsets;
- if (FAIL == (_memspace_id = H5Screate_simple(1, &count, NULL)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
- if (H5Sselect_all(_memspace_id) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
- if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, &count, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if (FAIL == (fastbit->offsets_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
- _space_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- /* Increment refcount so that anonymous dataset is persistent */
- if (FAIL == H5Oincr_refcount(fastbit->offsets_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
- }
-
- else if (typeIndex == BITMAPS_) {
- count = (hsize_t)gatherInfo[mpi_rank].nbitmaps;
- if (FAIL == (_memspace_id = H5Screate_simple(1, &count, NULL)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
- if (H5Sselect_all(_memspace_id) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
- if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, &count, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if (FAIL == (fastbit->bitmaps_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
- _space_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- /* Increment refcount so that anonymous dataset is persistent */
- if (FAIL == H5Oincr_refcount(fastbit->bitmaps_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
- }
- else if (typeIndex == INFO_) {
- if (FAIL == (_memspace_id = H5Screate_simple(1, array_size, NULL)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple (mem) dataspace");
- if (H5Sselect_all(_memspace_id) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
- if ((H5Sselect_hyperslab(_space_id, H5S_SELECT_SET, offset, NULL, array_size, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if (FAIL == (fastbit->index_info_group_id = H5Dcreate_anon(fastbit->file_id, fastbit->opaque_type_id,
- _space_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- /* Increment refcount so that anonymous dataset is persistent */
- if (FAIL == H5Oincr_refcount(fastbit->index_info_group_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
- }
-
- /* Save the outputs for use in writing the actual dataset! */
- *space_id = _space_id;
- *memspace_id = _memspace_id;
-
- done:
-
- FUNC_LEAVE_NOAPI(ret_value)
-}
-
-/*-------------------------------------------------------------------------
- * Function: H5X__fastbit_build_parallel_index
- *
- * Purpose: Same as H5X__fastbit_build_index, but parallel.
- * CALLED ONLY FROM THE SERIAL CODE
- * (when H5Xparallel_queries_enabled returns TRUE)
- * This is an attempt to modularize the code and
- * to make the mode more readable.
- *
- * Return: Non-negative on success/Negative on failure
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5X__fastbit_build_parallel_index(H5X_fastbit_t *fastbit)
-{
- static int sequence_id = 0;
- char log_entry[256];
-
- hid_t info_space_id = FAIL;
- hid_t key_memspace_id = H5S_ALL;
- hid_t offset_memspace_id = H5S_ALL;
- hid_t bitmap_memspace_id = H5S_ALL;
-
- hid_t keys_filespace_id = FAIL, offsets_filespace_id = FAIL, bitmaps_filespace_id = FAIL;
- hid_t keys_memspace_id = FAIL, offsets_memspace_id = FAIL, bitmaps_memspace_id = FAIL;
-
- hsize_t key_array_size, offset_array_size, bitmap_array_size;
- herr_t ret_value = SUCCEED; /* Return value */
- hid_t dxpl_id = H5P_DEFAULT;
- int i, mpi_rank, mpi_size;
- hsize_t filespace_offset = 0;
- hsize_t nkeys_totalsize = 0, nkeys_offset = 0;
- hsize_t noffsets_totalsize = 0, noffsets_offset = 0;
- hsize_t nbitmaps_totalsize = 0, nbitmaps_offset = 0;
- hsize_t indexinfo_totalsize = 0, indexinfo_offset = 0;
- struct fastbit_meta_collection *gatherInfo = NULL;
-
- FUNC_ENTER_NOAPI_NOINIT
-
- H5X_FASTBIT_LOG_DEBUG("Calling FastBit build parallel index");
- /* Build our process local index */
- if (0 != fastbit_iapi_build_index(fastbit->column_name, (const char *)0))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTENCODE, FAIL, "FastBit build index failed");
-
- /* Get process local arrays from index */
- if (0 != fastbit_iapi_deconstruct_index(fastbit->column_name, &fastbit->keys, &fastbit->nkeys,
- &fastbit->offsets, &fastbit->noffsets, &fastbit->bitmaps, &fastbit->nbitmaps))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "Can't get FastBit index arrays");
-
- H5X_FASTBIT_LOG_DEBUG("FastBit build parallel index, nkeys=%lu, noffsets=%lu, nbitmaps=%lu\n",
- fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
-
- mpi_size = H5Xparallel_size();
- mpi_rank = H5Xparallel_rank();
-
- /* We need storage for 3 elements per rank:: {nkeys, noffsets, n_bitmaps}
- *
- * +-------------+
- * | (RANK 0) |
- * | nkeys |
- * | noffsets |
- * | nbitmaps |
- * +-------------+
- * | |
- * ~ ~
- * +-------------+
- * | (RANK N) |
- * | nkeys |
- * | noffsets |
- * | nbitmaps |
- * +-------------+
- *
- */
- /* Allocate storage for 4 elements per rank */
- indexinfo_totalsize = (hsize_t)(sizeof(struct fastbit_meta_collection) * (hsize_t)(mpi_size));
- if (NULL == (gatherInfo = (struct fastbit_meta_collection *)H5MM_malloc(indexinfo_totalsize)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate nkeys array");
-
- gatherInfo[mpi_rank].data_length = (hsize_t)H5Sget_select_npoints(fastbit->memspace_id);
- gatherInfo[mpi_rank].nkeys = (hsize_t)(fastbit->nkeys);
- gatherInfo[mpi_rank].noffsets = (hsize_t)(fastbit->noffsets);
- gatherInfo[mpi_rank].nbitmaps = (hsize_t)(fastbit->nbitmaps);
-
- /* Exchange info with all MPI ranks */
- /* We could use allreduce here if we only needed the total sizes
- * but we also want to figure out the offsets (for the hyperslab selection).
- */
- if (H5Xallgather_by_size(gatherInfo, 4, sizeof(int64_t)) != SUCCEED)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't gather nkeys array");
-
- /* Gather the total sizes and offset info */
- for(i=0; i < mpi_size; i++) {
- if (i == mpi_rank) {
- nkeys_offset = nkeys_totalsize;
- noffsets_offset = noffsets_totalsize;
- nbitmaps_offset = nbitmaps_totalsize;
- fastbit->filespace_offset = filespace_offset;
- }
- nkeys_totalsize += gatherInfo[i].nkeys+1;
- noffsets_totalsize += gatherInfo[i].noffsets+1;
- nbitmaps_totalsize += gatherInfo[i].nbitmaps+1;
- filespace_offset += gatherInfo[i].data_length;
- }
-
- /* Save the exchanged info for future use */
- fastbit->nblocks = mpi_size;
- fastbit->idx_multiblock = TRUE;
- fastbit->nelmts = gatherInfo[mpi_rank].data_length;
-
- /* Create a transfer property to utilize MPI-IO */
- if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't define dataset_xfer property");
- if ((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't set dataset_xfer to MPIO_COLLECTIVE");
-
- /* -------------- */
- /* INFOGROUP */
- /* -------------- */
-
- /* Filespace descriptors */
- indexinfo_totalsize /= sizeof(int64_t); /* Previously, was the malloc size in bytes */
- /* Fixed to be the total number of elements */
- info_space_id = H5Screate_simple(1, &indexinfo_totalsize, NULL);
- keys_filespace_id = H5Screate_simple(1, &nkeys_totalsize, NULL);
- offsets_filespace_id = H5Screate_simple(1, &noffsets_totalsize, NULL);
- bitmaps_filespace_id = H5Screate_simple(1, &nbitmaps_totalsize, NULL);
-
- /* Memspace descriptors */
- keys_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nkeys, NULL);
- offsets_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].noffsets, NULL);
- bitmaps_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nbitmaps, NULL);
-
- if (H5Sselect_all(keys_memspace_id) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
- if (H5Sselect_all(offsets_memspace_id) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
- if (H5Sselect_all(bitmaps_memspace_id) < 0)
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "invalid memory space selection");
-
- /* Hyperslab selections */
- if ((H5Sselect_hyperslab(keys_filespace_id, H5S_SELECT_SET, (hsize_t *)(&nkeys_offset), NULL,
- (hsize_t *)(&gatherInfo[mpi_rank].nkeys), NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if ((H5Sselect_hyperslab(offsets_filespace_id, H5S_SELECT_SET, (hsize_t *)(&noffsets_offset), NULL,
- (hsize_t *)(&gatherInfo[mpi_rank].noffsets), NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if ((H5Sselect_hyperslab(bitmaps_filespace_id, H5S_SELECT_SET, &nbitmaps_offset, NULL,
- &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
-
- /* --------------------- */
- /* Close open datasets */
- /* --------------------- */
- if (fastbit->index_info_group_id != FAIL) {
- if (FAIL == H5Odecr_refcount(fastbit->index_info_group_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't decrement dataset refcount");
- if (FAIL == H5Dclose(fastbit->index_info_group_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close dataset");
- }
- if (fastbit->keys_id != FAIL) {
- if (FAIL == H5Odecr_refcount(fastbit->keys_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't decrement dataset refcount");
- if (FAIL == H5Dclose(fastbit->keys_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close dataset");
- }
- if (fastbit->offsets_id != FAIL) {
- if (FAIL == H5Odecr_refcount(fastbit->offsets_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't decrement dataset refcount");
- if (FAIL == H5Dclose(fastbit->offsets_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close dataset");
- }
- if (fastbit->bitmaps_id != FAIL) {
- if (FAIL == H5Odecr_refcount(fastbit->bitmaps_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't decrement dataset refcount");
- if (FAIL == H5Dclose(fastbit->bitmaps_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close dataset");
- }
- /* ---------------------- */
- /* New Anonymous datasets */
- /* ---------------------- */
-
- /* info */
- if (FAIL == (fastbit->index_info_group_id = H5Dcreate_anon(fastbit->file_id, H5T_NATIVE_LLONG, info_space_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- if (FAIL == H5Oincr_refcount(fastbit->index_info_group_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
-
- /* keys */
- if (FAIL == (fastbit->keys_id = H5Dcreate_anon(fastbit->file_id, H5T_NATIVE_DOUBLE, keys_filespace_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- if (FAIL == H5Oincr_refcount(fastbit->keys_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
-
- /* offsets */
- if (FAIL == (fastbit->offsets_id = H5Dcreate_anon(fastbit->file_id, H5T_NATIVE_LLONG, offsets_filespace_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- if (FAIL == H5Oincr_refcount(fastbit->offsets_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
-
- /* bitmaps */
- if (FAIL == (fastbit->bitmaps_id = H5Dcreate_anon(fastbit->file_id, H5T_NATIVE_UINT, bitmaps_filespace_id, H5P_DEFAULT, H5P_DEFAULT)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
- if (FAIL == H5Oincr_refcount(fastbit->bitmaps_id))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTINC, FAIL, "can't increment dataset refcount");
-
- /* ---------------------- */
- /* Write data to datasets */
- /* ---------------------- */
-
- /* Write parallel info */
- if (FAIL == H5Dwrite(fastbit->index_info_group_id, H5T_NATIVE_LLONG, H5S_ALL,
- H5S_ALL, dxpl_id, gatherInfo))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't write index metadata");
-
- /* Write keys */
- if (FAIL == H5Dwrite(fastbit->keys_id, H5T_NATIVE_DOUBLE, keys_memspace_id,
- keys_filespace_id, dxpl_id, fastbit->keys))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't write index metadata");
- /* Write offsets */
- if (FAIL == H5Dwrite(fastbit->offsets_id, H5T_NATIVE_LLONG, offsets_memspace_id,
- offsets_filespace_id, dxpl_id, fastbit->offsets))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't write index metadata");
-
- /* Write bitmaps */
- if (FAIL == H5Dwrite(fastbit->bitmaps_id, H5T_NATIVE_UINT, bitmaps_memspace_id,
- bitmaps_filespace_id, dxpl_id, fastbit->bitmaps))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't write index metadata");
-
-
- if (sequence_id == -1) { /* Disabled for now */
- key_array_size = gatherInfo[mpi_rank].nkeys * sizeof(double);
- sprintf(log_entry, "%d-keys-save-%s", sequence_id, fastbit->column_name);
- dump_as_bytes(log_entry, key_array_size, fastbit->keys);
-
- offset_array_size = gatherInfo[mpi_rank].noffsets * sizeof(int64_t);
- sprintf(log_entry, "%d-offsets-save-%s", sequence_id, fastbit->column_name);
- dump_as_bytes(log_entry, offset_array_size, fastbit->offsets);
-
- bitmap_array_size = gatherInfo[mpi_rank].nbitmaps * sizeof(uint32_t);
- sprintf(log_entry, "%d-bitmaps-save-%s", sequence_id, fastbit->column_name);
- dump_as_bytes(log_entry, bitmap_array_size, fastbit->bitmaps);
-
- sequence_id++;
- }
-done:
- if (info_space_id != FAIL)
- H5Sclose(info_space_id);
-
- if (keys_filespace_id != FAIL)
- H5Sclose(keys_filespace_id);
- if (offsets_filespace_id != FAIL)
- H5Sclose(offsets_filespace_id);
- if (bitmaps_filespace_id != FAIL)
- H5Sclose(bitmaps_filespace_id);
-
- if (keys_memspace_id != FAIL)
- H5Sclose(keys_memspace_id);
- if (offsets_memspace_id != FAIL)
- H5Sclose(offsets_memspace_id);
- if (bitmaps_memspace_id != FAIL)
- H5Sclose(bitmaps_memspace_id);
-
- if (dxpl_id != H5P_DEFAULT)
- H5Pclose(dxpl_id);
- if (err_occurred) {
- H5MM_free(gatherInfo);
- H5MM_free(fastbit->keys);
- fastbit->keys = NULL;
- H5MM_free(fastbit->offsets);
- fastbit->offsets = NULL;
- H5MM_free(fastbit->bitmaps);
- fastbit->bitmaps = NULL;
- }
- FUNC_LEAVE_NOAPI(ret_value)
-} /* H5X__fastbit_build_index */
-
-#endif /* H5_HAVE_PARALLEL */
-
/*-------------------------------------------------------------------------
* Function: H5X__fastbit_merge_data
*
@@ -1254,6 +646,7 @@ H5X__fastbit_scatter_cb(const void **src_buf/*out*/, size_t *src_buf_bytes_used/
void *_info)
{
struct H5X_fastbit_scatter_info *info = (struct H5X_fastbit_scatter_info *) _info;
+
/* Set output variables */
*src_buf = info->src_buf;
*src_buf_bytes_used = info->src_buf_size;
@@ -1279,42 +672,18 @@ H5X__fastbit_merge_data(H5X_fastbit_t *fastbit, const void *data,
if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get type from dataset");
+ if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
+ HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
+ if (0 == (nelmts_data = (size_t) H5Sget_select_npoints(dataspace_id)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
if (0 == (data_elmt_size = H5Tget_size(type_id)))
HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "invalid size of element");
-#ifdef H5_HAVE_PARALLEL
- /* For parallel operations, the dataspace_id being provided is one which
- * the defines the local view of the on-disk file, i.e. a hyperslab with
- * MPI rank specific offsets. Here, we've already
- * written data to the file and need to merge the local data with what
- * what previously there. We're merging these buffers in memory
- * so we don't want to deal with the offsets that were needed to write the
- * local slice onto disk.
- */
- if (H5Xparallel_queries_enabled() > 0) {
- if (fastbit->memspace_id != FAIL)
- dataspace_id = fastbit->memspace_id;
- nelmts_data = (size_t) H5Sget_select_npoints(dataspace_id);
- // printf("select_npoints(fastbit->filespace): nelmts = %llu, data_elmt_size = %llu\n", nelmts_data, data_elmt_size);
-
- info.src_buf = data;
- info.src_buf_size = nelmts_data * data_elmt_size;
- if (FAIL == H5Dscatter(H5X__fastbit_scatter_cb, &info, type_id, dataspace_id, buf))
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "cannot scatter data");
-
- }
- else
-#else
- {
- if (0 == (nelmts_data = (size_t) H5Sget_select_npoints(dataspace_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+ info.src_buf = data;
+ info.src_buf_size = nelmts_data * data_elmt_size;
- info.src_buf = data;
- info.src_buf_size = nelmts_data * data_elmt_size;
- if (FAIL == H5Dscatter(H5X__fastbit_scatter_cb, &info, type_id, dataspace_id, buf))
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "cannot scatter data");
- }
-#endif
+ if (FAIL == H5Dscatter(H5X__fastbit_scatter_cb, &info, type_id, dataspace_id, buf))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "cannot scatter data");
done:
if (type_id != FAIL)
@@ -1335,51 +704,34 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5X__fastbit_serialize_metadata(H5X_fastbit_t *fastbit, void *buf, size_t *buf_size)
+H5X__fastbit_serialize_metadata(H5X_fastbit_t *fastbit, void *buf,
+ size_t *buf_size)
{
- /* The implementation records 3 values (for the anonymous datasets)
- * {keys_id, offsets_id, bitmaps_id}
- * For parallel indexing/querying, we add an additional dataset (index_info_group).
- */
- int extra_info = 0;
- size_t metadata_size = 3 * sizeof(int64_t);
+ size_t metadata_size = 3 * sizeof(haddr_t);
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
-#ifdef H5_HAVE_PARALLEL
- if (H5Xparallel_queries_enabled() > 0) {
- extra_info = 1;
- metadata_size += sizeof(int64_t);
- }
-#endif
if (buf) {
H5O_info_t dset_info;
char *buf_ptr = (char *) buf;
dset_info.addr = 0;
- /* Maybe encode info_group */
- if (extra_info) {
- if ((fastbit->index_info_group_id != (hid_t)FAIL) && (FAIL == H5Oget_info(fastbit->index_info_group_id, &dset_info)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get info for anonymous dataset");
- HDmemcpy(buf_ptr, &dset_info.addr, sizeof(haddr_t));
- buf_ptr += sizeof(haddr_t);
- }
/* Encode keys info */
- if ((fastbit->keys_id != (hid_t)FAIL) && (FAIL == H5Oget_info(fastbit->keys_id, &dset_info)))
+ if ((fastbit->keys_id != FAIL) && (FAIL == H5Oget_info(fastbit->keys_id, &dset_info)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get info for anonymous dataset");
HDmemcpy(buf_ptr, &dset_info.addr, sizeof(haddr_t));
buf_ptr += sizeof(haddr_t);
/* Encode offsets info */
- if ((fastbit->offsets_id != (hid_t)FAIL) && (FAIL == H5Oget_info(fastbit->offsets_id, &dset_info)))
+ if ((fastbit->offsets_id != FAIL) && (FAIL == H5Oget_info(fastbit->offsets_id, &dset_info)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get info for anonymous dataset");
HDmemcpy(buf_ptr, &dset_info.addr, sizeof(haddr_t));
buf_ptr += sizeof(haddr_t);
/* Encode bitmaps info */
- if ((fastbit->bitmaps_id != (hid_t)FAIL) && (FAIL == H5Oget_info(fastbit->bitmaps_id, &dset_info)))
+ if ((fastbit->bitmaps_id != FAIL) && (FAIL == H5Oget_info(fastbit->bitmaps_id, &dset_info)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get info for anonymous dataset");
HDmemcpy(buf_ptr, &dset_info.addr, sizeof(haddr_t));
buf_ptr += sizeof(haddr_t);
@@ -1410,15 +762,6 @@ H5X__fastbit_deserialize_metadata(H5X_fastbit_t *fastbit, void *buf)
FUNC_ENTER_NOAPI_NOINIT
-#ifdef H5_HAVE_PARALLEL
- if (H5Xparallel_queries_enabled() > 0) {
- addr = *((haddr_t *) buf_ptr);
- if (addr && (FAIL == (fastbit->index_info_group_id = H5Oopen_by_addr(fastbit->file_id, *((haddr_t *) buf_ptr)))))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTOPENOBJ, FAIL, "can't open anonymous dataset");
- buf_ptr += sizeof(haddr_t);
- }
-#endif
-
/* Decode keys info */
addr = *((haddr_t *) buf_ptr);
if (addr && (FAIL == (fastbit->keys_id = H5Oopen_by_addr(fastbit->file_id, addr))))
@@ -1435,7 +778,6 @@ H5X__fastbit_deserialize_metadata(H5X_fastbit_t *fastbit, void *buf)
addr = *((haddr_t *) buf_ptr);
if (addr && (FAIL == (fastbit->bitmaps_id = H5Oopen_by_addr(fastbit->file_id, *((haddr_t *) buf_ptr)))))
HGOTO_ERROR(H5E_INDEX, H5E_CANTOPENOBJ, FAIL, "can't open anonymous dataset");
- buf_ptr += sizeof(haddr_t);
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -1464,340 +806,88 @@ H5X__fastbit_read_bitmaps(void *context, uint64_t start, uint64_t count,
}
static herr_t
-H5X__fastbit_reconstruct_parallel_index(H5X_fastbit_t *fastbit)
+H5X__fastbit_reconstruct_index(H5X_fastbit_t *fastbit)
{
- static int sequence_id = 0;
- char log_entry[256];
-
- hid_t dxpl_id = H5P_DEFAULT;
hid_t keys_space_id = FAIL, offsets_space_id = FAIL, bitmaps_space_id = FAIL;
- hid_t keys_filespace_id = FAIL, offsets_filespace_id = FAIL, bitmaps_filespace_id = FAIL;
- hid_t keys_memspace_id = FAIL, offsets_memspace_id = FAIL, bitmaps_memspace_id = FAIL;
-
size_t key_array_size, offset_array_size, bitmap_array_size;
FastBitDataType fastbit_type;
hid_t type_id = FAIL, space_id = FAIL;
- size_t nelmts, memelmts;
-
+ size_t nelmts;
herr_t ret_value = SUCCEED; /* Return value */
- hsize_t filespace_offset = 0;
- hsize_t nkeys_totalsize = 0, nkeys_offset = 0;
- hsize_t noffsets_totalsize = 0, noffsets_offset = 0;
- hsize_t nbitmaps_totalsize = 0, nbitmaps_offset = 0;
-
- struct fastbit_meta_collection *gatherInfo = NULL;
- hid_t info_space_id = FAIL;
- size_t info_array_size;
- size_t i, nranks;
- int mpi_size = H5Xparallel_size();
- int mpi_rank = H5Xparallel_rank();
FUNC_ENTER_NOAPI_NOINIT
- if ((fastbit->keys == NULL) ||
- (fastbit->offsets == NULL) ||
- (fastbit->bitmaps == NULL)) {
-
-
- /* Create a transfer property to utilize MPI-IO */
- if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't define dataset_xfer property");
- if ((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't set dataset_xfer to MPIO_COLLECTIVE");
-
- /* Read the spaceinfo from the anonymous dataset.
- * We will compare the computed total size per private store size
- * to ensure that things are ok to proceed.
- */
- if (FAIL == (info_space_id = H5Dget_space(fastbit->index_info_group_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
- /* Note that the npoints is NOT referenced as the number of bytes...
- * It refers to the number of elements that the dataset was written with.
- * In our case, we wrote the data as H5T_NATIVE_LLONG, so we'll get the number
- * of bytes for our calculation of nranks.
- */
- if (0 == (info_array_size = (size_t) H5Sget_select_npoints(info_space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
- if (NULL == (gatherInfo = (struct fastbit_meta_collection *) H5MM_malloc(info_array_size * sizeof(int64_t))))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate gatherInfo");
- if (FAIL == H5Dread(fastbit->index_info_group_id, H5T_NATIVE_LLONG,
- H5S_ALL, info_space_id, dxpl_id, gatherInfo))
- HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read gatherInfo data");
-
- /* The gatherInfo array is N groups of 3 int64_t values
- * It should be a simple thing to divide the total dataset size by
- * the size of a single entry to compute the number of rank entries
- * stored. This needs to match the current MPI size.
- */
- nranks = (info_array_size * sizeof(int64_t)) / sizeof(struct fastbit_meta_collection);
-
- for(i=0; i< nranks; i++) {
- if (i == mpi_rank) {
- nkeys_offset = nkeys_totalsize;
- noffsets_offset = noffsets_totalsize;
- nbitmaps_offset = nbitmaps_totalsize;
- fastbit->nelmts = (int64_t)(gatherInfo[i].data_length);
- fastbit->filespace_offset = filespace_offset;
- }
- nkeys_totalsize += gatherInfo[i].nkeys +1;
- noffsets_totalsize += gatherInfo[i].noffsets +1;
- nbitmaps_totalsize += gatherInfo[i].nbitmaps +1;
- filespace_offset += gatherInfo[i].data_length;
- }
-
- fastbit->nblocks = mpi_size;
- fastbit->idx_multiblock = TRUE;
- fastbit->nelmts = gatherInfo[mpi_rank].data_length;
-
- /* Prepare to read nkeys */
- if (FAIL == (keys_space_id = H5Dget_space(fastbit->keys_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
- if (0 == (key_array_size = (size_t) H5Sget_select_npoints(keys_space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-
- HDassert(nkeys_totalsize == key_array_size);
- fastbit->nkeys = gatherInfo[mpi_rank].nkeys;
-
- /* Prepare to read noffsets */
- if (FAIL == (offsets_space_id = H5Dget_space(fastbit->offsets_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
- if (0 == (offset_array_size = (size_t) H5Sget_select_npoints(offsets_space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-
- HDassert(noffsets_totalsize == offset_array_size);
- fastbit->noffsets = gatherInfo[mpi_rank].noffsets;
-
- /* Prepare to read nbitmaps */
- if (FAIL == (bitmaps_space_id = H5Dget_space(fastbit->bitmaps_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
- if (0 == (bitmap_array_size = (size_t) H5Sget_select_npoints(bitmaps_space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-
- HDassert(nbitmaps_totalsize == bitmap_array_size);
- fastbit->nbitmaps = gatherInfo[mpi_rank].nbitmaps;
-
- if (NULL == (fastbit->keys = (double *) H5MM_malloc(gatherInfo[mpi_rank].nkeys * sizeof(double))))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate keys");
- if (NULL == (fastbit->offsets = (int64_t *) H5MM_malloc(gatherInfo[mpi_rank].noffsets * sizeof(int64_t))))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
- if (NULL == (fastbit->bitmaps = (uint32_t *) H5MM_malloc(gatherInfo[mpi_rank].nbitmaps * sizeof(uint32_t))))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
-
- /* define the hyperslab selection(s) */
- if((keys_filespace_id = H5Screate_simple(1, &nkeys_totalsize, NULL)) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
- if((offsets_filespace_id = H5Screate_simple(1, &noffsets_totalsize, NULL)) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
- if((bitmaps_filespace_id = H5Screate_simple(1, &nbitmaps_totalsize, NULL)) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
-
- /* Memspace descriptors */
- if((keys_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nkeys, NULL)) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
- if((offsets_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].noffsets, NULL)) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
- if((bitmaps_memspace_id = H5Screate_simple(1, &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "H5Screate_simple returned an error");
-
- if (H5Sselect_all(keys_memspace_id) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "invalid memory space selection");
- if (H5Sselect_all(offsets_memspace_id) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "invalid memory space selection");
- if (H5Sselect_all(bitmaps_memspace_id) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "invalid memory space selection");
-
-
- if ((H5Sselect_hyperslab(keys_filespace_id, H5S_SELECT_SET, &nkeys_offset, NULL, &gatherInfo[mpi_rank].nkeys, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if ((H5Sselect_hyperslab(offsets_filespace_id, H5S_SELECT_SET, &noffsets_offset, NULL, &gatherInfo[mpi_rank].noffsets, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
- if ((H5Sselect_hyperslab(bitmaps_filespace_id, H5S_SELECT_SET, &nbitmaps_offset, NULL, &gatherInfo[mpi_rank].nbitmaps, NULL)) < 0 )
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't select hyperslab offsets");
-
- /* Read keys */
- if (FAIL == H5Dread(fastbit->keys_id, H5T_NATIVE_DOUBLE, keys_memspace_id,
- keys_filespace_id, dxpl_id, fastbit->keys))
- HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index keys metadata");
-
- /* Read offsets */
- if (FAIL == H5Dread(fastbit->offsets_id, H5T_NATIVE_LLONG, offsets_memspace_id,
- offsets_filespace_id, dxpl_id, fastbit->offsets))
- HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index offsets metadata");
-
- /* Read bitmaps */
- if (FAIL == H5Dread(fastbit->bitmaps_id, H5T_NATIVE_UINT, bitmaps_memspace_id,
- bitmaps_filespace_id, dxpl_id, fastbit->bitmaps))
- HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read index bitmaps metadata");
-
- if (sequence_id == -1) { /* Disabled for now */
- key_array_size = gatherInfo[mpi_rank].nkeys * sizeof(double);
- sprintf(log_entry, "%d-keys-read-%s", sequence_id, fastbit->column_name);
- dump_as_bytes(log_entry, key_array_size, fastbit->keys);
-
- offset_array_size = gatherInfo[mpi_rank].noffsets * sizeof(int64_t);
- sprintf(log_entry, "%d-offsets-read-%s", sequence_id, fastbit->column_name);
- dump_as_bytes(log_entry, offset_array_size, fastbit->offsets);
-
- bitmap_array_size = gatherInfo[mpi_rank].nbitmaps * sizeof(uint32_t);
- sprintf(log_entry, "%d-bitmaps-read-%s", sequence_id, fastbit->column_name);
- dump_as_bytes(log_entry, bitmap_array_size, fastbit->bitmaps);
-
- sequence_id++;
- }
-
-
- /* Reconstruct index */
- H5X_FASTBIT_LOG_DEBUG("Reconstructing index with nkeys=%lu, noffsets=%lu, "
- "nbitmaps=%lu", fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
-
- /* Get space info */
- if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
- HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get type from dataset");
- if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
-
- /* We've restored the number of elements in the gatherInfo array (.data_length)
- * and cached it for use here and possibly elsewhere.
- */
- nelmts = (size_t)fastbit->nelmts;
-
- /* Convert type to FastBit type */
- if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
-
- /* Register array */
- if (0 != fastbit_iapi_register_array_index_only(fastbit->column_name,
- fastbit_type, &nelmts, 1, fastbit->keys, fastbit->nkeys,
- fastbit->offsets, fastbit->noffsets, fastbit->bitmaps,
- H5X__fastbit_read_bitmaps))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTREGISTER, FAIL, "can't register array");
-
- H5X_FASTBIT_LOG_DEBUG("Reconstructed index");
- fastbit->idx_reconstructed = TRUE;
- }
-
-done:
+ /* TODO don't read keys and offsets if already present */
+ HDassert(!fastbit->keys);
+ HDassert(!fastbit->offsets);
- if (FAIL != type_id)
- H5Tclose(type_id);
- if (dxpl_id != H5P_DEFAULT)
- H5Pclose(dxpl_id);
- if (FAIL != keys_space_id)
- H5Sclose(keys_space_id);
- if (FAIL != offsets_space_id)
- H5Sclose(offsets_space_id);
- if (FAIL != bitmaps_space_id)
- H5Sclose(bitmaps_space_id);
- if (info_space_id != FAIL)
- H5Sclose(info_space_id);
- if (keys_filespace_id != FAIL)
- H5Sclose(keys_filespace_id);
- if (offsets_filespace_id != FAIL)
- H5Sclose(offsets_filespace_id);
- if (bitmaps_filespace_id != FAIL)
- H5Sclose(bitmaps_filespace_id);
-
- if (keys_memspace_id != FAIL)
- H5Sclose(keys_memspace_id);
- if (offsets_memspace_id != FAIL)
- H5Sclose(offsets_memspace_id);
- if (bitmaps_memspace_id != FAIL)
- H5Sclose(bitmaps_memspace_id);
+ if (FAIL == (keys_space_id = H5Dget_space(fastbit->keys_id)))
+ HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+ if (0 == (key_array_size = (size_t) H5Sget_select_npoints(keys_space_id)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+ fastbit->nkeys = key_array_size / sizeof(double);
- FUNC_LEAVE_NOAPI(ret_value);
-}
+ if (FAIL == (offsets_space_id = H5Dget_space(fastbit->offsets_id)))
+ HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+ if (0 == (offset_array_size = (size_t) H5Sget_select_npoints(offsets_space_id)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+ fastbit->noffsets = offset_array_size / sizeof(int64_t);
-static herr_t
-H5X__fastbit_reconstruct_index(H5X_fastbit_t *fastbit)
-{
- static int sequence_id = 0;
- char log_entry[256];
- hid_t keys_space_id = FAIL, offsets_space_id = FAIL, bitmaps_space_id = FAIL;
- size_t key_array_size, offset_array_size, bitmap_array_size;
- FastBitDataType fastbit_type;
- hid_t type_id = FAIL, space_id = FAIL;
- size_t nelmts;
- herr_t ret_value = SUCCEED; /* Return value */
+ if (FAIL == (bitmaps_space_id = H5Dget_space(fastbit->bitmaps_id)))
+ HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+ if (0 == (bitmap_array_size = (size_t) H5Sget_select_npoints(bitmaps_space_id)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+ fastbit->nbitmaps = bitmap_array_size / sizeof(uint32_t);
+
+ /* allocate buffer to hold data */
+ if (NULL == (fastbit->keys = (double *) H5MM_malloc(key_array_size)))
+ HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate keys");
+ if (NULL == (fastbit->offsets = (int64_t *) H5MM_malloc(offset_array_size)))
+ HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
+ if (NULL == (fastbit->bitmaps = (uint32_t *) H5MM_malloc(bitmap_array_size)))
+ HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
+
+ /* Read FastBit keys */
+ if (FAIL == H5Dread(fastbit->keys_id, fastbit->opaque_type_id,
+ H5S_ALL, keys_space_id, H5P_DEFAULT, fastbit->keys))
+ HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
-#ifdef H5_HAVE_PARALLEL
- if (H5Xparallel_queries_enabled() > 0)
- // if (fastbit->idx_multiblock)
- return H5X__fastbit_reconstruct_parallel_index(fastbit);
-#endif
- FUNC_ENTER_NOAPI_NOINIT
+ /* Read FastBit offsets */
+ if (FAIL == H5Dread(fastbit->offsets_id, fastbit->opaque_type_id,
+ H5S_ALL, offsets_space_id, H5P_DEFAULT, fastbit->offsets))
+ HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
- /* TODO don't read keys and offsets if already present */
- if ((fastbit->keys == NULL) ||
- (fastbit->offsets == NULL) ||
- (fastbit->bitmaps == NULL)) {
+ /* Read FastBit bitmaps */
+ if (FAIL == H5Dread(fastbit->bitmaps_id, fastbit->opaque_type_id,
+ H5S_ALL, bitmaps_space_id, H5P_DEFAULT, fastbit->bitmaps))
+ HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
- if (FAIL == (keys_space_id = H5Dget_space(fastbit->keys_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
- if (0 == (key_array_size = (size_t) H5Sget_select_npoints(keys_space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
- fastbit->nkeys = key_array_size / sizeof(double);
+ /* Reconstruct index */
+ H5X_FASTBIT_LOG_DEBUG("Reconstructing index with nkeys=%lu, noffsets=%lu, "
+ "nbitmaps=%lu", fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
- if (FAIL == (offsets_space_id = H5Dget_space(fastbit->offsets_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
- if (0 == (offset_array_size = (size_t) H5Sget_select_npoints(offsets_space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
- fastbit->noffsets = offset_array_size / sizeof(int64_t);
+ /* Get space info */
+ if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
+ HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get type from dataset");
+ if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
+ if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
- if (FAIL == (bitmaps_space_id = H5Dget_space(fastbit->bitmaps_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
- if (0 == (bitmap_array_size = (size_t) H5Sget_select_npoints(bitmaps_space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
- fastbit->nbitmaps = bitmap_array_size / sizeof(uint32_t);
-
- /* allocate buffer to hold data */
- if (NULL == (fastbit->keys = (double *) H5MM_malloc(key_array_size)))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate keys");
- if (NULL == (fastbit->offsets = (int64_t *) H5MM_malloc(offset_array_size)))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
- if (NULL == (fastbit->bitmaps = (uint32_t *) H5MM_malloc(bitmap_array_size)))
- HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate offsets");
-
- /* Read FastBit keys */
- if (FAIL == H5Dread(fastbit->keys_id, fastbit->opaque_type_id,
- H5S_ALL, keys_space_id, H5P_DEFAULT, fastbit->keys))
- HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
-
- /* Read FastBit offsets */
- if (FAIL == H5Dread(fastbit->offsets_id, fastbit->opaque_type_id,
- H5S_ALL, offsets_space_id, H5P_DEFAULT, fastbit->offsets))
- HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
-
- /* Read FastBit bitmaps */
- if (FAIL == H5Dread(fastbit->bitmaps_id, fastbit->opaque_type_id,
- H5S_ALL, bitmaps_space_id, H5P_DEFAULT, fastbit->bitmaps))
- HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
-
- /* Reconstruct index */
- H5X_FASTBIT_LOG_DEBUG("Reconstructing index with nkeys=%lu, noffsets=%lu, "
- "nbitmaps=%lu", fastbit->nkeys, fastbit->noffsets, fastbit->nbitmaps);
-
- /* Get space info */
- if (FAIL == (type_id = H5Dget_type(fastbit->dataset_id)))
- HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get type from dataset");
- if (FAIL == (space_id = H5Dget_space(fastbit->dataset_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
- if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
- HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+ /* Convert type to FastBit type */
+ if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
+ HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
- /* Convert type to FastBit type */
- if (FastBitDataTypeUnknown == (fastbit_type = H5X__fastbit_convert_type(type_id)))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTCONVERT, FAIL, "can't convert type");
+ /* Register array */
+ if (0 != fastbit_iapi_register_array_index_only(fastbit->column_name,
+ fastbit_type, &nelmts, 1, fastbit->keys, fastbit->nkeys,
+ fastbit->offsets, fastbit->noffsets, fastbit->bitmaps,
+ H5X__fastbit_read_bitmaps))
+ HGOTO_ERROR(H5E_INDEX, H5E_CANTREGISTER, FAIL, "can't register array");
- /* Register array */
- if (0 != fastbit_iapi_register_array_index_only(fastbit->column_name,
- fastbit_type, &nelmts, 1, fastbit->keys, fastbit->nkeys,
- fastbit->offsets, fastbit->noffsets, fastbit->bitmaps,
- H5X__fastbit_read_bitmaps))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTREGISTER, FAIL, "can't register array");
+ H5X_FASTBIT_LOG_DEBUG("Reconstructed index");
+ fastbit->idx_reconstructed = TRUE;
- H5X_FASTBIT_LOG_DEBUG("Reconstructed index");
- fastbit->idx_reconstructed = TRUE;
- }
done:
if (FAIL != type_id)
H5Tclose(type_id);
@@ -1906,43 +996,43 @@ H5X__fastbit_evaluate_query(H5X_fastbit_t *fastbit, hid_t query_id,
cv = H5X__fasbit_query_get_value_as_double(query);
selection_handle = fastbit_selection_osr(fastbit->column_name, compare_type, cv);
- H5X_FASTBIT_LOG_DEBUG("Selection estimate: %ld", fastbit_selection_estimate(selection_handle));
+ H5X_FASTBIT_LOG_DEBUG("Selection estimate: %ld",
+ fastbit_selection_estimate(selection_handle));
+#if 0
if (0 > (nhits = fastbit_selection_evaluate(selection_handle)))
HGOTO_ERROR(H5E_INDEX, H5E_CANTCOMPUTE, FAIL, "can't evaluate selection");
+#else
+ nhits = fastbit_selection_evaluate(selection_handle);
+#endif
- if (nhits > 0) {
+ if (nhits) {
if (NULL == (hit_coords = (uint64_t *) H5MM_malloc(((size_t) nhits) * sizeof(uint64_t))))
HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate hit coordinates");
- if (0 > fastbit_selection_get_coordinates(selection_handle, hit_coords,(uint64_t) nhits, 0U))
+ if (0 > fastbit_selection_get_coordinates(selection_handle, hit_coords,
+ (uint64_t) nhits, 0U))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get coordinates");
#ifdef H5X_FASTBIT_DEBUG
/* TODO remove debug */
{
- int k;
- int64_t loops = nhits/10;
- int64_t i;
- printf(" # %s(): Index read contains following %lld coords: ", __func__, nhits);
- for (i=0 ; i < loops; i += 10) {
- printf("\n");
- for(k=0; k< 10; k++) {
- printf("%lu ", hit_coords[i+k]);
- }
- }
+ int i;
+ printf(" # %s(): Index read contains following coords: ", __func__);
+ for (i = 0; i < nhits; i++) {
+ printf("%lu ", hit_coords[i]);
+ }
printf("\n");
}
#endif
}
-
*coords = hit_coords;
*ncoords = (uint64_t) nhits;
done:
if (selection_handle)
fastbit_selection_free(selection_handle);
- if ((FAIL == ret_value) && (hit_coords != NULL)) {
+ if (FAIL == ret_value) {
H5MM_free(hit_coords);
}
FUNC_LEAVE_NOAPI(ret_value)
@@ -1955,9 +1045,6 @@ done:
*
* Return: Non-negative on success/Negative on failure
*
- * RAW:: This needs to be able to create a NULL selection so that we don't
- * FAIL unless there is an actual error. It should NOT be an error
- * to not have any matches (i.e. when ncords == 0)!
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1967,65 +1054,35 @@ H5X__fastbit_create_selection(H5X_fastbit_t *fastbit, hid_t dataspace_id,
hsize_t count[H5S_MAX_RANK + 1];
hsize_t start_coord[H5S_MAX_RANK + 1], end_coord[H5S_MAX_RANK + 1], nelmts;
herr_t ret_value = SUCCEED; /* Return value */
- size_t i, dataset_rank;
- uint64_t colimit = ncoords;
-
- double t_start, t_end, t_loop, t_select_bounds, t_select_npoints;
- double t_hyperslab = 0, temp, t0, t1;
- double t_min = 0.1, t_max = 0, target = 0.01;
+ size_t i;
FUNC_ENTER_NOAPI_NOINIT
- /* Check for non-null selections */
- if (ncoords > 0) {
+ /* Initialize count */
+ for (i = 0; i < H5S_MAX_RANK; i++)
+ count[i] = 1;
+
+ for (i = 0; i < ncoords; i++) {
hsize_t new_coords[H5S_MAX_RANK + 1];
- HDmemset(new_coords, 0, sizeof(new_coords));
- /* Initialize count */
-#ifdef LIMIT_RESULTS_FOR_TESTING
- if (colimit > 10000) colimit = 10000;
-#endif
- for (i = 0; i < H5S_MAX_RANK; i++)
- count[i] = 1;
+ const hsize_t point = coords[i];
-#ifdef SHOW_TIME
- t_start = MPI_Wtime();
-#endif
- for (i = 0; i < colimit /* ncoords*/; i++) {
- const hsize_t point = coords[i] + fastbit->filespace_offset;
- /* Convert coordinates */
- if (FAIL == H5VM_array_calc_pre(point, fastbit->dataset_ndims, fastbit->dataset_down_dims, new_coords))
- HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate coord array");
-
- /* Add converted coordinate to selection */
- if (H5Sselect_hyperslab(dataspace_id, H5S_SELECT_OR, new_coords, NULL, count, NULL))
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "unable to add point to selection");
- }
+ /* Convert coordinates */
+ if (FAIL == H5VM_array_calc_pre(point, fastbit->dataset_ndims,
+ fastbit->dataset_down_dims, new_coords))
+ HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate coord array");
-#ifdef SHOW_TIME
- t_end = MPI_Wtime();
- t_loop = (t_end - t_start);
- t_start = t_end;
-#endif
+ /* Add converted coordinate to selection */
+ if (H5Sselect_hyperslab(dataspace_id, H5S_SELECT_OR, new_coords, NULL, count, NULL))
+ HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "unable to add point to selection");
+ }
+ if (ncoords) { /* Shouldn't report errors of zero(0) query matches */
if (FAIL == H5Sget_select_bounds(dataspace_id, start_coord, end_coord))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSELECT, FAIL, "unable to get bounds");
-#ifdef SHOW_TIME
- t_end = MPI_Wtime();
- t_select_bounds = (t_end - t_start);
- t_start = t_end;
-#endif
if (0 == (nelmts = (hsize_t) H5Sget_select_npoints(dataspace_id)))
HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-#ifdef SHOW_TIME
- t_end = MPI_Wtime();
- t_select_npoints = (t_end - t_start);
-
- printf("\tt_loop = %f (cummulative)\n", t_loop);
- printf("\tt_select_bounds = %f\n", t_select_bounds);
- printf("\tt_select_npoints = %f\n", t_select_npoints);
-#endif
H5X_FASTBIT_LOG_DEBUG("Created dataspace from index with %llu elements [(%llu, %llu):(%llu, %llu)]",
- nelmts, start_coord[0], start_coord[1], end_coord[0], end_coord[1]);
+ nelmts, start_coord[0], start_coord[1], end_coord[0], end_coord[1]);
}
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -2047,11 +1104,10 @@ H5X_fastbit_create(hid_t dataset_id, hid_t xcpl_id, hid_t H5_ATTR_UNUSED xapl_id
{
H5X_fastbit_t *fastbit = NULL;
void *ret_value = NULL; /* Return value */
- size_t private_metadata_size = 0;
+ size_t private_metadata_size;
void *buf = NULL;
size_t buf_size;
hbool_t read_on_create = TRUE;
- double t_start, t_end;
FUNC_ENTER_NOAPI_NOINIT
H5X_FASTBIT_LOG_DEBUG("Enter");
@@ -2068,27 +1124,17 @@ H5X_fastbit_create(hid_t dataset_id, hid_t xcpl_id, hid_t H5_ATTR_UNUSED xapl_id
if (read_on_create) {
/* Get data from dataset */
-#ifdef SHOW_TIME
- t_start = MPI_Wtime();
-#endif
if (FAIL == H5X__fastbit_read_data(dataset_id, fastbit->column_name, &buf, &buf_size))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get data from dataset");
-#ifdef SHOW_TIME
- t_end = MPI_Wtime();
- printf("H5X__fastbit_read_data finished reading %lld bytes in %f seconds\n", buf_size, t_end -t_start);
+
/* Index data */
- t_start = t_end;
-#endif
if (FAIL == H5X__fastbit_build_index(fastbit))
HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, NULL, "can't create index data from dataset");
-#ifdef SHOW_TIME
- t_end = MPI_Wtime();
- printf("H5X__fastbit_build_index completed in %f seconds\n", t_end -t_start);
-#endif
}
/* Serialize metadata for H5X interface */
- if (FAIL == H5X__fastbit_serialize_metadata(fastbit, NULL, &private_metadata_size))
+ if (FAIL == H5X__fastbit_serialize_metadata(fastbit, NULL,
+ &private_metadata_size))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get plugin metadata size");
if (NULL == (fastbit->private_metadata = H5MM_malloc(private_metadata_size)))
@@ -2131,6 +1177,7 @@ H5X_fastbit_remove(hid_t file_id, size_t metadata_size, void *metadata)
FUNC_ENTER_NOAPI_NOINIT
H5X_FASTBIT_LOG_DEBUG("Enter");
+
if (metadata_size < (3 * sizeof(haddr_t)))
HGOTO_ERROR(H5E_INDEX, H5E_BADVALUE, FAIL, "metadata size is not valid");
@@ -2227,7 +1274,7 @@ H5X_fastbit_close(void *idx_handle)
FUNC_ENTER_NOAPI_NOINIT
H5X_FASTBIT_LOG_DEBUG("Enter");
-
+
if (NULL == fastbit)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL index handle");
@@ -2249,20 +1296,17 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5X_fastbit_pre_update(void *idx_handle, hid_t filespace_id, hid_t H5_ATTR_UNUSED xxpl_id)
+H5X_fastbit_pre_update(void *idx_handle, hid_t H5_ATTR_UNUSED dataspace_id, hid_t H5_ATTR_UNUSED xxpl_id)
{
H5X_fastbit_t *fastbit = (H5X_fastbit_t *) idx_handle;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
H5X_FASTBIT_LOG_DEBUG("Enter");
+
if (NULL == fastbit)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL index handle");
-#ifdef H5_HAVE_PARALLEL
- fastbit->filespace_id = filespace_id;
-#endif
-
done:
H5X_FASTBIT_LOG_DEBUG("Leave");
FUNC_LEAVE_NOAPI(ret_value)
@@ -2289,6 +1333,7 @@ H5X_fastbit_post_update(void *idx_handle, const void *data, hid_t dataspace_id,
FUNC_ENTER_NOAPI_NOINIT
H5X_FASTBIT_LOG_DEBUG("Calling H5X_fastbit_post_update");
+
if (NULL == fastbit)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL index handle");
@@ -2305,7 +1350,7 @@ H5X_fastbit_post_update(void *idx_handle, const void *data, hid_t dataspace_id,
HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create index data from dataset");
done:
- H5MM_free(buf);
+// H5MM_free(buf);
H5X_FASTBIT_LOG_DEBUG("Leave");
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5X_fastbit_post_update() */
@@ -2329,7 +1374,6 @@ H5X_fastbit_query(void *idx_handle, hid_t H5_ATTR_UNUSED dataspace_id,
uint64_t *coords = NULL;
uint64_t ncoords = 0;
hid_t ret_value = FAIL; /* Return value */
- double t_start, t_end;
FUNC_ENTER_NOAPI_NOINIT
H5X_FASTBIT_LOG_DEBUG("Enter");
@@ -2349,31 +1393,18 @@ H5X_fastbit_query(void *idx_handle, hid_t H5_ATTR_UNUSED dataspace_id,
if (FAIL == H5Sselect_none(ret_space_id))
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSELECT, FAIL, "can't reset selection");
-#ifdef SHOW_TIME
- t_start = MPI_Wtime();
-#endif
-
/* Get matching coordinates values from query */
if (FAIL == H5X__fastbit_evaluate_query(fastbit, query_id, &coords, &ncoords))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get query range");
-#ifdef SHOW_TIME
- t_end = MPI_Wtime();
- printf("H5X__fastbit_evaluate_query took %f seconds to process %lld of %lld total elements, resulting in %lld coords\n", t_end - t_start, fastbit->nelmts, fastbit->dataset_dims[0], ncoords);
- t_start = t_end;
-#endif
+
/* Create selection */
if (FAIL == H5X__fastbit_create_selection(fastbit, ret_space_id, coords, ncoords))
HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get query range");
-#ifdef SHOW_TIME
- t_end = MPI_Wtime();
- printf("H5X__fastbit_create_selection took %f seconds\n", t_end - t_start);
-#endif
ret_value = ret_space_id;
done:
- if (coords != NULL)
- H5MM_free(coords);
+ H5MM_free(coords);
if (FAIL != space_id)
H5Sclose(space_id);
if ((FAIL == ret_value) && (FAIL != ret_space_id))
diff --git a/src/H5Xmeta_db.c b/src/H5Xmeta_db.c
index c1d7799..24f8798 100644
--- a/src/H5Xmeta_db.c
+++ b/src/H5Xmeta_db.c
@@ -36,13 +36,30 @@
#define H5R_FRIEND
#include "H5Rpkg.h" /* (Tmp) To re-use H5R__get_obj_name */
+#if defined(__linux__)
+#include <linux/limits.h>
+#define H5X_DB_MAX_NAME_LEN PATH_MAX
+#endif
+
#ifdef H5_HAVE_DB
#include <db.h>
/****************/
/* Local Macros */
/****************/
-//#define H5X_DB_DEBUG
+// #define H5X_DB_DEBUG
+#define H5X_DB_SHOW_TIMING_AND_STATS
+#ifdef H5X_DB_SHOW_TIMING_AND_STATS
+#define H5X_DB_LOG_INFO(...) do { \
+ fprintf(stdout, " # %s(): ", __func__); \
+ fprintf(stdout, __VA_ARGS__); \
+ fprintf(stdout, "\n"); \
+ fflush(stdout); \
+ } while (0)
+#else
+#define H5X_DB_LOG_INFO(...) do { \
+ } while (0)
+#endif
#ifdef H5X_DB_DEBUG
#define H5X_DB_LOG_DEBUG(...) do { \
@@ -142,7 +159,10 @@ struct { \
//#define H5X_DB_METADATA_TYPES 3
#define H5X_DB_NPRIMARIES 2
#define H5X_DB_NSECONDARIES 4
+
+#ifndef H5X_DB_MAX_NAME_LEN
#define H5X_DB_MAX_NAME_LEN (64 * 1024)
+#endif
#define H5X_DB_OBJ_REFS_SUFFIX "__obj_refs.db"
#define H5X_DB_LINK_NAMES_SUFFIX "__link_names.db"
@@ -312,6 +332,7 @@ const H5X_class_t H5X_META_DB[1] = {{
H5X_PLUGIN_META_DB, /* (Or whatever number is assigned) */
"DB", /* Whatever name desired */
H5X_TYPE_METADATA, /* This plugin operates on metadata */
+ H5X_SIMPLE_QUERY, /* Query types */
&idx_class /* Index class */
}};
@@ -402,7 +423,7 @@ H5X__db_insert_obj(H5X_db_t *db, href_t ref, H5X_db_obj_t *obj)
{
DBT key; /* The key to dbcp->put() */
DBT data; /* The data to dbcp->put() */
- size_t ref_buf_size;
+ size_t ref_buf_size = 0;
void *ref_buf = NULL;
herr_t ret_value = SUCCEED; /* Return value */
int db_ret;
@@ -433,7 +454,8 @@ H5X__db_insert_obj(H5X_db_t *db, href_t ref, H5X_db_obj_t *obj)
}
done:
- H5MM_free(ref_buf);
+ if (ref_buf)
+ H5MM_free(ref_buf);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5X__db_insert_obj */
@@ -542,12 +564,12 @@ H5X__db_index_attr(hid_t loc_id, const char *attr_name,
if (FAIL == H5X__db_insert_attr(args->db, ref, &db_attr))
HGOTO_ERROR(H5E_QUERY, H5E_CANTAPPEND, FAIL, "can't append object reference to view");
-// H5X_db_index_attr_elem_arg_t iter_args;
+ // H5X_db_index_attr_elem_arg_t iter_args;
// iter_args.attr_args = args;
// iter_args.attr_name = attr_name;
/* Iterate over attribute elements to compare values */
-// if (FAIL == H5Diterate(buf, type_id, space_id, H5X__db_index_attr_value_iterate, &iter_args))
-// HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOMPARE, FAIL, "unable to compare attribute elements");
+ // if (FAIL == H5Diterate(buf, type_id, space_id, H5X__db_index_attr_value_iterate, &iter_args))
+ // HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOMPARE, FAIL, "unable to compare attribute elements");
done:
if (ref && FAIL == H5Rdestroy(ref))
@@ -1165,10 +1187,8 @@ H5X__db_stat(H5X_db_t *db)
db->attr_names_file, db->attr_link_names_file, db->attr_values_file };
DB *sdbs[H5X_DB_NSECONDARIES] = { db->link_names_db, db->attr_names_db,
db->attr_link_names_db, db->attr_values_db };
-#ifdef H5X_DB_DEBUG
const char *sdb_names[H5X_DB_NSECONDARIES] = { "link names", "attr names",
"attr link names", "attr values" };
-#endif
int i;
/* Stat secondaries */
@@ -1196,6 +1216,11 @@ H5X__db_stat(H5X_db_t *db)
H5X_DB_LOG_DEBUG("Current database size on disk: %lf MB",
(float)st.st_size / (float)(1024 * 1024));
+ H5X_DB_LOG_INFO("Database contains %lu records for %s",
+ (u_long)statp->bt_ndata, sdb_names[i]);
+ H5X_DB_LOG_INFO("Current database size on disk: %lf MB",
+ (float)st.st_size / (float)(1024 * 1024));
+
HDfree(statp);
}
}
@@ -1306,7 +1331,7 @@ H5X__db_create(hid_t loc_id, hid_t H5_ATTR_UNUSED xcpl_id,
hid_t H5_ATTR_UNUSED xapl_id, size_t *metadata_size, void **metadata)
{
H5X_db_t *db = NULL;
-#ifdef H5X_DB_DEBUG
+#ifdef H5X_DB_SHOW_TIMING_AND_STATS
struct timeval t1, t2;
#endif
void *ret_value = NULL; /* Return value */
@@ -1336,21 +1361,22 @@ H5X__db_create(hid_t loc_id, hid_t H5_ATTR_UNUSED xcpl_id,
HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, NULL, "can't create tables");
/* Visit file */
-#ifdef H5X_DB_DEBUG
+#ifdef H5X_DB_SHOW_TIMING_AND_STATS
HDgettimeofday(&t1, NULL);
#endif
if (FAIL == H5Ovisit(loc_id, H5_INDEX_NAME, H5_ITER_NATIVE, H5X__db_index, db))
HGOTO_ERROR(H5E_SYM, H5E_BADITER, NULL, "object visitation failed");
-#ifdef H5X_DB_DEBUG
+
+#ifdef H5X_DB_SHOW_TIMING_AND_STATS
HDgettimeofday(&t2, NULL);
#endif
- H5X_DB_LOG_DEBUG("######################################################");
- H5X_DB_LOG_DEBUG("Index build time: %lf ms",
+ H5X_DB_LOG_INFO("######################################################");
+ H5X_DB_LOG_INFO("Index build time: %lf ms",
((float)(t2.tv_sec - t1.tv_sec)) * 1000.0f
+ ((float)(t2.tv_usec - t1.tv_usec)) / (1000.0f));
H5X__db_stat(db);
- H5X_DB_LOG_DEBUG("######################################################");
+ H5X_DB_LOG_INFO("######################################################");
/* Write index metadata */
if (FAIL == H5X__db_metadata_write(db, metadata_size, metadata))