Modify indexing example to first write dataset / create index / do query

Move index creation code to H5X_alacrity_create / no longer use set_extent
Fix ALACRITY metadata read
Clean up

2 files changed, 390 insertions, 214 deletions

diff --git a/src/H5Xalacrity.c b/src/H5Xalacrity.c
index 2e43888..37581f4 100644
--- a/src/H5Xalacrity.c
+++ b/src/H5Xalacrity.c
@@ -105,10 +105,6 @@ static herr_t
 H5X_alacrity_query(void *idx_handle, hid_t query_id, hid_t xxpl_id,
         hid_t *dataspace_id);
 
-static herr_t
-H5X__alacrity_get_query_data_cb(void *elem, hid_t type_id, unsigned ndim,
-        const hsize_t *point, void *_udata);
-
 /*********************/
 /* Package Variables */
 /*********************/
@@ -136,6 +132,243 @@ const H5X_class_t H5X_ALACRITY[1] = {{
     H5X_alacrity_query         /* query */
 }};
 
+static herr_t
+H5X__alacrity_configure(H5X_alacrity_t *alacrity, hid_t dataset_id)
+{
+    hid_t dataset_type_id;
+    size_t dataset_type_size;
+    ALDatatype alacrity_type;
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT
+
+    if (FAIL == (alacrity->opaque_type_id = H5Tcreate(H5T_OPAQUE, 1)))
+        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCREATE, FAIL, "can't create type");
+    if (FAIL == H5Tset_tag(alacrity->opaque_type_id, "alacrity metadata type"))
+        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, FAIL, "can't set tag to type");
+
+    if (FAIL == (dataset_type_id = H5Dget_type(dataset_id)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get type from dataset");
+    if (0 == (dataset_type_size = H5Tget_size(dataset_type_id)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get type size");
+    printf("Type size: %zu\n", dataset_type_size);
+    alacrity_type = (dataset_type_size == 4) ? DATATYPE_FLOAT32 : DATATYPE_FLOAT64;
+
+    ALEncoderConfigure(&alacrity->config, 16, alacrity_type, ALInvertedIndex);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+}
+
+static herr_t
+H5X__alacrity_get_dataset_data(hid_t file_id, hid_t dataset_id, hid_t trans_id,
+        void **buf, size_t *buf_size)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+    hid_t rcxt_id, type_id, space_id;
+    size_t nelmts, elmt_size;
+    uint64_t version;
+
+    FUNC_ENTER_NOAPI_NOINIT
+
+    /* Get read context */
+    if (FAIL == H5TRget_version(trans_id, &version))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get version from transaction ID");
+    if (FAIL == (rcxt_id =  H5RCcreate(file_id, version)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create read context");
+
+    /* Get space info */
+    if (FAIL == (type_id = H5Dget_type(dataset_id)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get type from dataset");
+    if (FAIL == (space_id = H5Dget_space(dataset_id)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
+    if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
+        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+    if (0 == (elmt_size = H5Tget_size(type_id)))
+        HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "invalid size of element");
+
+    printf("*** npoints: %zu\n", (size_t) nelmts);
+
+    /* Allocate buffer to hold data */
+    *buf_size = nelmts * elmt_size;
+    if(NULL == (*buf = H5MM_malloc(*buf_size)))
+        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
+
+    /* Read data from dataset */
+    if (FAIL == H5Dread_ff(dataset_id, type_id, H5S_ALL, space_id,
+            H5P_DEFAULT, *buf, rcxt_id, H5_EVENT_STACK_NULL))
+        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+
+    /* Close read context */
+    if (FAIL == H5RCclose(rcxt_id))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCLOSEOBJ, FAIL, "can't close read context");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+}
+
+static herr_t
+H5X__alacrity_free_data(void *buf)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+    free(buf);
+
+    FUNC_LEAVE_NOAPI(ret_value)
+}
+
+static herr_t
+H5X__alacrity_create_index(H5X_alacrity_t *alacrity, hid_t file_id, hid_t dataset_id,
+        hid_t trans_id, const void *buf, size_t buf_size)
+{
+    hid_t type_id, space_id;
+    hid_t metadata_space_id, data_space_id, index_space_id;
+    hsize_t metadata_size, index_size, data_size;
+    memstream_t memstream;
+    size_t nelmts;
+    void *metadata_buf;
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT
+
+    /* Get space info */
+    if (FAIL == (type_id = H5Dget_type(dataset_id)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get type from dataset");
+    if (FAIL == (space_id = H5Dget_space(dataset_id)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
+    if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
+        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+
+    ALEncode(&alacrity->config, buf, nelmts, &alacrity->output);
+
+    /* Get sizes */
+    metadata_size = ALGetMetadataSize(&alacrity->output.metadata);
+    data_size = ALGetDataSize(&alacrity->output.data,
+            &alacrity->output.metadata);
+    index_size = ALGetIndexSize(&alacrity->output.index,
+            &alacrity->output.metadata);
+
+    printf("*** Metadata size: %zu\n", (size_t) metadata_size);
+    printf("*** Data size: %zu\n", (size_t) data_size);
+    printf("*** Index size: %zu\n", (size_t) index_size);
+
+    /* Create metadata array with opaque type */
+    if (FAIL == (metadata_space_id = H5Screate_simple(1, &metadata_size, NULL)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple dataspace");
+
+    if (FAIL == (alacrity->metadata_id = H5Dcreate_anon_ff(file_id, alacrity->opaque_type_id,
+            metadata_space_id, H5P_DEFAULT, H5P_DEFAULT, trans_id, H5_EVENT_STACK_NULL)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
+
+    /* Create data array with opaque type */
+    if (FAIL == (data_space_id = H5Screate_simple(1, &data_size, NULL)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple dataspace");
+
+    if (FAIL == (alacrity->data_id = H5Dcreate_anon_ff(file_id, alacrity->opaque_type_id,
+            data_space_id, H5P_DEFAULT, H5P_DEFAULT, trans_id, H5_EVENT_STACK_NULL)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
+
+    /* Create index array with opaque type */
+    if (FAIL == (index_space_id = H5Screate_simple(1, &index_size, NULL)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create simple dataspace");
+
+    if (FAIL == (alacrity->index_id = H5Dcreate_anon_ff(file_id, alacrity->opaque_type_id,
+            index_space_id, H5P_DEFAULT, H5P_DEFAULT, trans_id, H5_EVENT_STACK_NULL)))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, FAIL, "can't create anonymous dataset");
+
+    /* Serialize and write ALACRITY metadata */
+    metadata_buf = H5MM_malloc(metadata_size);
+    if (NULL == metadata_buf)
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate index elements");
+    memstreamInit(&memstream, metadata_buf);
+    ALSerializeMetadata(&alacrity->output.metadata, &memstream);
+    if (FAIL == H5Dwrite_ff(alacrity->metadata_id, alacrity->opaque_type_id, H5S_ALL,
+            H5S_ALL, H5P_DEFAULT, memstream.buf, trans_id, H5_EVENT_STACK_NULL))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't update index metadata");
+    memstreamDestroy(&memstream, 0);
+
+    /* Write ALACRITY compressed data */
+    if (FAIL == H5Dwrite_ff(alacrity->data_id, alacrity->opaque_type_id, H5S_ALL,
+            H5S_ALL, H5P_DEFAULT, alacrity->output.data, trans_id, H5_EVENT_STACK_NULL))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't update index data");
+
+    /* Write ALACRITY index */
+    if (FAIL == H5Dwrite_ff(alacrity->index_id, alacrity->opaque_type_id, H5S_ALL,
+            H5S_ALL, H5P_DEFAULT, alacrity->output.index, trans_id, H5_EVENT_STACK_NULL))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't update index data");
+done:
+    H5MM_free(metadata_buf);
+//    if (metadata_space_id != FAIL) H5Sclose(metadata_space_id);
+    FUNC_LEAVE_NOAPI(ret_value)
+}
+
+static herr_t
+H5X__alacrity_serialize_metadata(H5X_alacrity_t *alacrity, void *buf,
+        size_t *buf_size)
+{
+    size_t metadata_token_size, data_token_size, index_token_size;
+    size_t alacrity_metadata_size;
+
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT
+
+    /* Get tokens */
+    if (FAIL == H5Oget_token(alacrity->metadata_id, NULL, &metadata_token_size))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get token size for anonymous dataset");
+    if (FAIL == H5Oget_token(alacrity->data_id, NULL, &data_token_size))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get token size for anonymous dataset");
+    if (FAIL == H5Oget_token(alacrity->index_id, NULL, &index_token_size))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get token size for anonymous dataset");
+
+    /* Make some space for the metadata (tokens + sizes) */
+    alacrity_metadata_size = metadata_token_size + data_token_size +
+            index_token_size + 3 * sizeof(size_t);
+
+    if (buf) {
+        char *buf_ptr = buf;
+
+        /* Encode metadata token info */
+        HDmemcpy(buf_ptr, &metadata_token_size, sizeof(size_t));
+        buf_ptr += sizeof(size_t);
+        if (FAIL == H5Oget_token(alacrity->metadata_id, buf_ptr, &metadata_token_size))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get token for anonymous dataset");
+        buf_ptr += metadata_token_size;
+
+        /* Encode data token info */
+        HDmemcpy(buf_ptr, &data_token_size, sizeof(size_t));
+        buf_ptr += sizeof(size_t);
+        if (FAIL == H5Oget_token(alacrity->data_id, buf_ptr, &data_token_size))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get token for anonymous dataset");
+        buf_ptr += data_token_size;
+
+        /* Encode index token info */
+        HDmemcpy(buf_ptr, &index_token_size, sizeof(size_t));
+        buf_ptr += sizeof(size_t);
+        if (FAIL == H5Oget_token(alacrity->index_id, buf_ptr, &index_token_size))
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get token for anonymous dataset");
+        buf_ptr += index_token_size;
+    }
+    if (buf_size) *buf_size = alacrity_metadata_size;
+
+ done:
+    FUNC_LEAVE_NOAPI(ret_value)
+}
+
+static herr_t
+H5X__alacrity_findBinRange1C(ALMetadata metadata, hid_t query_id,
+        bin_id_t *start_bin, bin_id_t *end_bin)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
 /*-------------------------------------------------------------------------
  * Function:    H5X_alacrity_create
  *
@@ -151,16 +384,13 @@ H5X_alacrity_create(hid_t file_id, hid_t dataset_id, hid_t UNUSED xcpl_id,
         hid_t xapl_id, size_t *metadata_size, void **metadata)
 {
     H5X_alacrity_t *alacrity = NULL;
-    hid_t space_id, trans_id;
-    hid_t dataset_type_id;
+    hid_t trans_id;
     void *ret_value = NULL; /* Return value */
-    hsize_t dsize = 1, dmax = H5S_UNLIMITED;
     size_t dataset_type_size;
-    ALDatatype alacrity_type;
-    size_t metadata_token_size, data_token_size, index_token_size;
     size_t alacrity_metadata_size;
     void *alacrity_metadata;
-    char *alacrity_metadata_ptr;
+    void *buf = NULL;
+    size_t buf_size;
 
     FUNC_ENTER_NOAPI_NOINIT
 
@@ -170,80 +400,36 @@ H5X_alacrity_create(hid_t file_id, hid_t dataset_id, hid_t UNUSED xcpl_id,
         HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, NULL, "can't allocate alacrity struct");
     alacrity->metadata_read = FALSE;
 
-    /* Configure the encoder depending on the size of the data */
-    if (FAIL == (alacrity->opaque_type_id = H5Tcreate(H5T_OPAQUE, (size_t)1)))
-        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCREATE, NULL, "can't create type");
-    if (FAIL == H5Tset_tag(alacrity->opaque_type_id, "alacrity metadata type"))
-        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set tag to type");
-
-    if (FAIL == (dataset_type_id = H5Dget_type(dataset_id)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get type from dataset");
-    dataset_type_size = H5Tget_size(dataset_type_id);
-    printf("Type size: %zu\n", dataset_type_size);
-    alacrity_type = (dataset_type_size == 4) ? DATATYPE_FLOAT32 : DATATYPE_FLOAT64;
-
-    /* Configure encoder */
-    ALEncoderConfigure(&alacrity->config, 16, alacrity_type, ALCompressionIndex);
+    /* Configure the ALACRITY encoder */
+    if (FAIL == H5X__alacrity_configure(alacrity, dataset_id))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, NULL, "can't configure ALACRITY");
 
 #ifdef H5_HAVE_INDEXING
     if (FAIL == H5Pget_xapl_transaction(xapl_id, &trans_id))
         HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get trans_id from xapl");
 #endif
 
-    /* Create a dataspace with unlimited size */
-    if ((space_id = H5Screate_simple(1, &dsize, &dmax)) < 0)
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, NULL, "can't create dataspace");
-
-    /* Create metadata array with opaque type */
-    if (FAIL == (alacrity->metadata_id = H5Dcreate_anon_ff(file_id, alacrity->opaque_type_id,
-            space_id, H5P_DEFAULT, H5P_DEFAULT, trans_id, H5_EVENT_STACK_NULL)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, NULL, "can't create anonymous dataset");
-
-    /* Create data array with opaque type */
-    if (FAIL == (alacrity->data_id = H5Dcreate_anon_ff(file_id, alacrity->opaque_type_id,
-            space_id, H5P_DEFAULT, H5P_DEFAULT, trans_id, H5_EVENT_STACK_NULL)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, NULL, "can't create anonymous dataset");
+    /* Get data from dataset */
+    if (FAIL == H5X__alacrity_get_dataset_data(file_id, dataset_id, trans_id,
+            &buf, &buf_size))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get data from dataset");
 
-    /* Create index array with opaque type */
-    if (FAIL == (alacrity->index_id = H5Dcreate_anon_ff(file_id, alacrity->opaque_type_id,
-            space_id, H5P_DEFAULT, H5P_DEFAULT, trans_id, H5_EVENT_STACK_NULL)))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTCREATE, NULL, "can't create anonymous dataset");
+    /* Index data */
+    if (FAIL == H5X__alacrity_create_index(alacrity, file_id, dataset_id,
+            trans_id, buf, buf_size))
+        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't index data from dataset");
 
-    /* Get tokens */
-    if (FAIL == H5Oget_token(alacrity->metadata_id, NULL, &metadata_token_size))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get token size for anonymous dataset");
-    if (FAIL == H5Oget_token(alacrity->data_id, NULL, &data_token_size))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get token size for anonymous dataset");
-    if (FAIL == H5Oget_token(alacrity->index_id, NULL, &index_token_size))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get token size for anonymous dataset");
+    /* Serialize metadata for H5X interface */
+    if (FAIL == H5X__alacrity_serialize_metadata(alacrity, NULL,
+            &alacrity_metadata_size))
+        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, NULL, "can't allocate token metadata");
 
-    /* Make some space for the metadata (tokens + sizes) */
-    alacrity_metadata_size = metadata_token_size + data_token_size +
-            index_token_size + 3 * sizeof(size_t);
     if (NULL == (alacrity_metadata = H5MM_malloc(alacrity_metadata_size)))
         HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, NULL, "can't allocate token metadata");
-    alacrity_metadata_ptr = alacrity_metadata;
 
-    /* Encode metadata token info */
-    HDmemcpy(alacrity_metadata_ptr, &metadata_token_size, sizeof(size_t));
-    alacrity_metadata_ptr += sizeof(size_t);
-    if (FAIL == H5Oget_token(alacrity->metadata_id, alacrity_metadata_ptr, &metadata_token_size))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get token for anonymous dataset");
-    alacrity_metadata_ptr += metadata_token_size;
-
-    /* Encode data token info */
-    HDmemcpy(alacrity_metadata_ptr, &data_token_size, sizeof(size_t));
-    alacrity_metadata_ptr += sizeof(size_t);
-    if (FAIL == H5Oget_token(alacrity->data_id, alacrity_metadata_ptr, &data_token_size))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get token for anonymous dataset");
-    alacrity_metadata_ptr += data_token_size;
-
-    /* Encode index token info */
-    HDmemcpy(alacrity_metadata_ptr, &index_token_size, sizeof(size_t));
-    alacrity_metadata_ptr += sizeof(size_t);
-    if (FAIL == H5Oget_token(alacrity->index_id, alacrity_metadata_ptr, &index_token_size))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, NULL, "can't get token for anonymous dataset");
-    alacrity_metadata_ptr += index_token_size;
+    if (FAIL == H5X__alacrity_serialize_metadata(alacrity, alacrity_metadata,
+            &alacrity_metadata_size))
+        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, NULL, "can't allocate token metadata");
 
     /* Metadata is token for anonymous dataset */
     *metadata = alacrity_metadata;
@@ -252,6 +438,7 @@ H5X_alacrity_create(hid_t file_id, hid_t dataset_id, hid_t UNUSED xcpl_id,
     ret_value = alacrity;
 
 done:
+    H5X__alacrity_free_data(buf);
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5X_alacrity_create() */
 
@@ -316,8 +503,11 @@ H5X_alacrity_open(hid_t file_id, hid_t UNUSED dataset_id, hid_t xapl_id,
         HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, NULL, "can't allocate alacrity struct");
     alacrity->metadata_read = FALSE;
 
-    /* TODO do not need opaque type in open for now */
-    alacrity->opaque_type_id = FAIL;
+    /* TODO move that */
+    if (FAIL == (alacrity->opaque_type_id = H5Tcreate(H5T_OPAQUE, 1)))
+        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCREATE, FAIL, "can't create type");
+    if (FAIL == H5Tset_tag(alacrity->opaque_type_id, "alacrity metadata type"))
+        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, FAIL, "can't set tag to type");
 
     /* TODO do that like this for now */
     H5RCget_version(rc_id, &c_version);
@@ -432,15 +622,10 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5X_alacrity_post_update(void *idx_handle, const void *buf, hid_t dataspace_id,
-        hid_t xxpl_id)
+H5X_alacrity_post_update(void *idx_handle, const void UNUSED *buf,
+        hid_t UNUSED dataspace_id, hid_t UNUSED xxpl_id)
 {
     H5X_alacrity_t *alacrity = (H5X_alacrity_t *) idx_handle;
-    hid_t trans_id;
-    hsize_t npoints;
-    hsize_t metadata_size, index_size, data_size;
-    void *metadata_buf = NULL;
-    memstream_t memstream; /* Alacrity Memstream */
     herr_t ret_value = SUCCEED; /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -450,69 +635,8 @@ H5X_alacrity_post_update(void *idx_handle, const void *buf, hid_t dataspace_id,
     if (NULL == alacrity)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL index handle");
 
-//    if (FAIL == (file_space_id = H5Dget_space(alacrity->anon_id)))
-//        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from dataset");
-
-#ifdef H5_HAVE_INDEXING
-    if (FAIL == H5Pget_xxpl_transaction(xxpl_id, &trans_id))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get trans_id from xxpl");
-#endif
-
-    npoints = (hsize_t) H5Sget_select_npoints(dataspace_id);
-    printf("*** npoints: %zu\n", (size_t) npoints);
-
-    ALEncode(&alacrity->config, buf, npoints, &alacrity->output);
-
-    /* Get sizes */
-    metadata_size = ALGetMetadataSize(&alacrity->output.metadata);
-    data_size = ALGetDataSize(&alacrity->output.data,
-            &alacrity->output.metadata);
-    index_size = ALGetIndexSize(&alacrity->output.index,
-            &alacrity->output.metadata);
-
-    printf("*** Metadata size: %zu\n", (size_t) metadata_size);
-    printf("*** Data size: %zu\n", (size_t) data_size);
-    printf("*** Index size: %zu\n", (size_t) index_size);
-
-    /* Extend metadata dataset */
-    if (FAIL == H5Dset_extent_ff(alacrity->metadata_id, &metadata_size,
-            trans_id, H5_EVENT_STACK_NULL))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, FAIL, "can't set extent");
-    if (FAIL == H5Dset_extent_ff(alacrity->data_id, &data_size,
-            trans_id, H5_EVENT_STACK_NULL))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, FAIL, "can't set extent");
-    if (FAIL == H5Dset_extent_ff(alacrity->index_id, &index_size,
-            trans_id, H5_EVENT_STACK_NULL))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, FAIL, "can't set extent");
-
-//    if (FAIL == (metadata_space_id = H5Dget_space(alacrity->metadata_id)))
-//        HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, FAIL, "can't get dataspace ID");
-
-    /* Serialize the metadata */
-    metadata_buf = H5MM_malloc(metadata_size);
-    if (NULL == metadata_buf)
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTALLOC, FAIL, "can't allocate index elements");
-
-    memstreamInit(&memstream, metadata_buf);
-    ALSerializeMetadata(&alacrity->output.metadata, &memstream);
-    if (FAIL == H5Dwrite_ff(alacrity->metadata_id, alacrity->opaque_type_id, H5S_ALL,
-            H5S_ALL, H5P_DEFAULT, memstream.buf, trans_id, H5_EVENT_STACK_NULL))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't update index metadata");
-    memstreamDestroy(&memstream, 0);
-
-    /* Write the data */
-    if (FAIL == H5Dwrite_ff(alacrity->data_id, alacrity->opaque_type_id, H5S_ALL,
-            H5S_ALL, H5P_DEFAULT, alacrity->output.data, trans_id, H5_EVENT_STACK_NULL))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't update index data");
-
-    /* Write the index */
-    if (FAIL == H5Dwrite_ff(alacrity->index_id, alacrity->opaque_type_id, H5S_ALL,
-            H5S_ALL, H5P_DEFAULT, alacrity->output.index, trans_id, H5_EVENT_STACK_NULL))
-        HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't update index data");
-
+    /* Not needed here */
 done:
-    H5MM_free(metadata_buf);
-//    if (metadata_space_id != FAIL) H5Sclose(metadata_space_id);
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5X_alacrity_post_update() */
 
@@ -566,13 +690,9 @@ H5X_alacrity_query(void *idx_handle, hid_t query_id, hid_t xxpl_id,
         hid_t *dataspace_id)
 {
     H5X_alacrity_t *alacrity = (H5X_alacrity_t *) idx_handle;
-    H5X__alacrity_query_data_t udata;
+    ALIndex alacrity_index;
     bin_id_t start_bin, end_bin;
-    hid_t space_id, type_id;
     hid_t rcxt_id;
-    size_t nelmts;
-    size_t elmt_size = 0, buf_size = 0;
-    void *buf = NULL;
     herr_t ret_value = SUCCEED; /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -588,20 +708,34 @@ H5X_alacrity_query(void *idx_handle, hid_t query_id, hid_t xxpl_id,
 
     if (!alacrity->metadata_read) {
         memstream_t memstream; /* Alacrity Memstream */
-        void *metadata_buf;
         hid_t metadata_space_id;
+        void *buf;
+        size_t buf_size;
 
         if (FAIL == (metadata_space_id = H5Dget_space(alacrity->metadata_id)))
-            HGOTO_ERROR(H5E_INDEX, H5E_CANTSET, FAIL, "can't get dataspace ID");
-        printf("**** npoints: %zu\n", H5Sget_select_npoints(metadata_space_id));
-
-        if (FAIL == H5Dread_ff(alacrity->metadata_id, alacrity->opaque_type_id, H5S_ALL,
-                H5S_ALL, H5P_DEFAULT, memstream.buf, rcxt_id, H5_EVENT_STACK_NULL))
-            HGOTO_ERROR(H5E_INDEX, H5E_CANTUPDATE, FAIL, "can't update index metadata");
-
-        memstreamInit(&memstream, metadata_buf);
+            HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
+        if (0 == (buf_size = (size_t) H5Sget_select_npoints(metadata_space_id)))
+            HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
+
+        /* allocate buffer to hold data */
+        if(NULL == (buf = H5MM_malloc(buf_size)))
+            HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
+        memset(buf, '\0', buf_size);
+
+        /* read metadata */
+        if (FAIL == H5Dread_ff(alacrity->metadata_id, alacrity->opaque_type_id,
+                H5S_ALL, metadata_space_id, H5P_DEFAULT, buf, rcxt_id,
+                H5_EVENT_STACK_NULL))
+            HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
+
+        memstreamInit(&memstream, buf);
         ALDeserializeMetadata(&alacrity->metadata, &memstream);
         memstreamDestroy(&memstream, 0);
+        printf("**** Partition length: %zu\n", alacrity->metadata.partitionLength);
+        H5MM_free(buf);
+
+        /* Set this to TRUE so we don't read metadata again */
+        alacrity->metadata_read = TRUE;
     }
 
     /**
@@ -609,31 +743,19 @@ H5X_alacrity_query(void *idx_handle, hid_t query_id, hid_t xxpl_id,
      * query range will fall into these bins, however not all elements in these
      * bins fall into the query range).
      */
-//    findBinRange1C(alacrity->metadata, query_id, &start_bin, &end_bin);
-
-    /* First findBinRange1C */
+    H5X__alacrity_findBinRange1C(alacrity->metadata, query_id, &start_bin, &end_bin);
 
     /* Second readIndex */
+    ALGetIndexBinOffset(alacrity->metadata, start_bin);
+    ALGetIndexBinOffset(alacrity->metadata, end_bin);
+
+    H5X__alacrity_readIndex(alacrity, start_bin, end_bin, &alacrity_index);
+
+    if (alacrity->metadata->indexMeta.indexForm == ALCompressedInvertedIndex)
+        ALConvertPartialIndexForm(&alacrity->metadata, &alacrity_index,
+                ALInvertedIndex, start_bin, end_bin);
+
 
-    //    if (FAIL == (type_id = H5Dget_type(alacrity->idx_anon_id)))
-//        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get type from index");
-//    if (FAIL == (space_id = H5Dget_space(alacrity->idx_anon_id)))
-//        HGOTO_ERROR(H5E_INDEX, H5E_CANTGET, FAIL, "can't get dataspace from index");
-//    if (0 == (nelmts = (size_t) H5Sget_select_npoints(space_id)))
-//        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of elements");
-//    if (0 == (elmt_size = H5Tget_size(type_id)))
-//        HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "invalid size of element");
-//
-//    /* allocate buffer to hold data */
-//    buf_size = nelmts * elmt_size;
-//    if(NULL == (buf = H5MM_malloc(buf_size)))
-//        HGOTO_ERROR(H5E_INDEX, H5E_NOSPACE, FAIL, "can't allocate read buffer");
-//
-//    /* read data from index */
-//    if (FAIL == H5Dread_ff(alacrity->idx_anon_id, type_id, H5S_ALL, space_id,
-//            H5P_DEFAULT, buf, rcxt_id, H5_EVENT_STACK_NULL))
-//        HGOTO_ERROR(H5E_INDEX, H5E_READERROR, FAIL, "can't read data");
-//
 //    if(FAIL == (udata.space_query = H5Scopy(space_id)))
 //        HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to copy dataspace");
 //    if(H5Sselect_none(udata.space_query) < 0)
@@ -652,6 +774,5 @@ H5X_alacrity_query(void *idx_handle, hid_t query_id, hid_t xxpl_id,
 //            (int) H5Sget_select_npoints(*dataspace_id));
 
 done:
-    H5MM_free(buf);
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5X_alacrity_query() */
diff --git a/test/index.c b/test/index.c
index f8ede76..b364d26 100644
--- a/test/index.c
+++ b/test/index.c
@@ -20,32 +20,30 @@ write_dataset(hid_t file_id, const char *dataset_name,
 {
     hid_t       dataset_id;
     hid_t       file_space_id, mem_space_id;
-    hid_t       tid1, rid1, trspl_id;
+    hid_t       trans_id, rcxt_id, trspl_id;
     hsize_t     dims[2] = {total, ncomponents};
     hsize_t     offset[2] = {start, 0};
     hsize_t     count[2] = {ntuples, ncomponents};
     int         rank = (ncomponents == 1) ? 1 : 2;
-    uint64_t    version;
+    uint64_t    version = 1;
     herr_t      ret;
 
     /* acquire container version 1 - EXACT. */
     if(0 == my_rank) {
-        version = 1;
-        rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+        rcxt_id = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+    } else {
+        rcxt_id = H5RCcreate(file_id, version);
     }
-    MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
     assert(1 == version);
-    if (my_rank != 0)
-        rid1 = H5RCcreate(file_id, version);
 
     /* create transaction object */
-    tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
-    assert(tid1);
+    trans_id = H5TRcreate(file_id, rcxt_id, version + 1);
+    assert(trans_id);
 
     trspl_id = H5Pcreate(H5P_TR_START);
     ret = H5Pset_trspl_num_peers(trspl_id, (unsigned int) my_size);
     assert(0 == ret);
-    ret = H5TRstart(tid1, trspl_id, estack_id);
+    ret = H5TRstart(trans_id, trspl_id, estack_id);
     assert(0 == ret);
     ret = H5Pclose(trspl_id);
     assert(0 == ret);
@@ -56,20 +54,15 @@ write_dataset(hid_t file_id, const char *dataset_name,
 
     /* Create a dataset. */
     dataset_id = H5Dcreate_ff(file_id, dataset_name, datatype_id, file_space_id,
-            H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, estack_id);
+            H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, trans_id, estack_id);
     assert(dataset_id);
 
-    /* Add indexing information */
-    ret = H5Xcreate_ff(file_id, H5X_PLUGIN_ALACRITY, dataset_id, H5P_DEFAULT,
-            tid1, estack_id);
-    assert(0 == ret);
-
     mem_space_id = H5Screate_simple(rank, count, NULL);
     assert(mem_space_id);
 
     /* Write the first dataset. */
     ret = H5Dwrite_ff(dataset_id, datatype_id, mem_space_id, file_space_id,
-            H5P_DEFAULT, buf, tid1, estack_id);
+            H5P_DEFAULT, buf, trans_id, estack_id);
     assert(0 == ret);
 
     /* Close the data space for the first dataset. */
@@ -84,19 +77,76 @@ write_dataset(hid_t file_id, const char *dataset_name,
     assert(0 == ret);
 
     /* Finish transaction 0. */
-    ret = H5TRfinish(tid1, H5P_DEFAULT, NULL, estack_id);
+    ret = H5TRfinish(trans_id, H5P_DEFAULT, NULL, estack_id);
+    assert(0 == ret);
+
+    /* release container version 0. */
+    if (my_rank == 0) {
+        ret = H5RCrelease(rcxt_id, estack_id);
+        assert(0 == ret);
+    }
+
+    ret = H5RCclose(rcxt_id);
+    assert(0 == ret);
+
+    ret = H5TRclose(trans_id);
+    assert(0 == ret);
+}
+
+static void
+create_index(hid_t file_id, const char *dataset_name, hid_t estack_id)
+{
+    hid_t dataset_id, trans_id, rcxt_id;
+    hid_t trspl_id;
+    uint64_t version = 2;
+    herr_t ret;
+
+    /* acquire container version 1 - EXACT. */
+    if(0 == my_rank) {
+        rcxt_id = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+    } else {
+        rcxt_id = H5RCcreate(file_id, version);
+    }
+    assert(2 == version);
+
+    /* create transaction object */
+    trans_id = H5TRcreate(file_id, rcxt_id, version + 1);
+    assert(trans_id);
+
+    trspl_id = H5Pcreate(H5P_TR_START);
+    ret = H5Pset_trspl_num_peers(trspl_id, (unsigned int) my_size);
+    assert(0 == ret);
+    ret = H5TRstart(trans_id, trspl_id, estack_id);
+    assert(0 == ret);
+    ret = H5Pclose(trspl_id);
+    assert(0 == ret);
+
+    dataset_id = H5Dopen_ff(file_id, dataset_name, H5P_DEFAULT, rcxt_id,
+            estack_id);
+
+    /* Add indexing information */
+    ret = H5Xcreate_ff(file_id, H5X_PLUGIN_ALACRITY, dataset_id, H5P_DEFAULT,
+            trans_id, estack_id);
+    assert(0 == ret);
+
+    /* Close the first dataset. */
+    ret = H5Dclose_ff(dataset_id, estack_id);
+    assert(0 == ret);
+
+    /* Finish transaction 0. */
+    ret = H5TRfinish(trans_id, H5P_DEFAULT, NULL, estack_id);
     assert(0 == ret);
 
     /* release container version 0. */
     if (my_rank == 0) {
-        ret = H5RCrelease(rid1, estack_id);
+        ret = H5RCrelease(rcxt_id, estack_id);
         assert(0 == ret);
     }
 
-    ret = H5RCclose(rid1);
+    ret = H5RCclose(rcxt_id);
     assert(0 == ret);
 
-    ret = H5TRclose(tid1);
+    ret = H5TRclose(trans_id);
     assert(0 == ret);
 }
 
@@ -108,8 +158,8 @@ query_and_view(hid_t file_id, const char *dataset_name, hid_t estack_id)
     hid_t  query_id1, query_id2, query_id3, query_id4, query_id5, query_id6;
     hid_t query_id, view_id;
     hid_t dataset_id;
-    hid_t rid2;
-    uint64_t version;
+    hid_t rcxt_id;
+    uint64_t version = 3;
     herr_t ret;
 
     /* Create a simple query */
@@ -140,32 +190,33 @@ query_and_view(hid_t file_id, const char *dataset_name, hid_t estack_id)
     assert(query_id);
 
     /* acquire container version 2 - EXACT. */
-    version = 2;
-    rid2 = H5RCacquire(file_id, &version, H5P_DEFAULT, estack_id);
-    assert(rid2 > 0);
-    assert(2 == version);
+    version = 3;
+    rcxt_id = H5RCacquire(file_id, &version, H5P_DEFAULT, estack_id);
+    assert(rcxt_id > 0);
+    assert(3 == version);
 
     MPI_Barrier(MPI_COMM_WORLD);
 
-    dataset_id = H5Dopen_ff(file_id, dataset_name, H5P_DEFAULT, rid2,
+    dataset_id = H5Dopen_ff(file_id, dataset_name, H5P_DEFAULT, rcxt_id,
             estack_id);
 
-    view_id = H5Vcreate_ff(dataset_id, query_id, H5P_DEFAULT, rid2,
-            estack_id);
-    assert(view_id > 0);
+//    view_id = H5Vcreate_ff(dataset_id, query_id, H5P_DEFAULT, rid2,
+//            estack_id);
+//    assert(view_id > 0);
+    H5Dquery_ff(dataset_id, query_id, -1, rcxt_id);
 
     /* TODO use view_id for analysis shipping etc */
 
-    H5Vclose(view_id);
+//    H5Vclose(view_id);
 
     ret = H5Dclose_ff(dataset_id, estack_id);
     assert(0 == ret);
 
     /* release container version 2. */
-    ret = H5RCrelease(rid2, estack_id);
+    ret = H5RCrelease(rcxt_id, estack_id);
     assert(0 == ret);
 
-    ret = H5RCclose(rid2);
+    ret = H5RCclose(rcxt_id);
     assert(0 == ret);
 
     H5Qclose(query_id);
@@ -237,6 +288,10 @@ main(int argc, char **argv)
 
     MPI_Barrier(MPI_COMM_WORLD);
 
+    create_index(file_id, dataset_name, estack_id);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+
     query_and_view(file_id, dataset_name, estack_id);
 
     MPI_Barrier(MPI_COMM_WORLD);