summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNeil Fortner <nfortne2@hdfgroup.org>2017-05-23 22:30:04 (GMT)
committerNeil Fortner <nfortne2@hdfgroup.org>2017-05-23 22:30:04 (GMT)
commit40d013a8696f782ee6d801b07e734e8f9a1f3dda (patch)
tree3089b367ae8e38dfedd595ea941d4c585089dd0e
parent91550e6015524e5f9915d767ade1bdcedb14010a (diff)
downloadhdf5-40d013a8696f782ee6d801b07e734e8f9a1f3dda.zip
hdf5-40d013a8696f782ee6d801b07e734e8f9a1f3dda.tar.gz
hdf5-40d013a8696f782ee6d801b07e734e8f9a1f3dda.tar.bz2
Add support for simple variable length datatypes in datasets (including
selections). Updated h5dsm_tvlen.c to test this. Other minor fixes/cleanup.
-rw-r--r--examples/h5dsm_tvlen.c439
-rw-r--r--src/H5VLdaosm.c845
2 files changed, 1078 insertions, 206 deletions
diff --git a/examples/h5dsm_tvlen.c b/examples/h5dsm_tvlen.c
index ffe4933..596db7a 100644
--- a/examples/h5dsm_tvlen.c
+++ b/examples/h5dsm_tvlen.c
@@ -80,18 +80,50 @@ error:
return -1;
} /* end check_vls() */
+#define UPDATE_FBUF_VLA(FBUF, UBUF) \
+do { \
+ if((FBUF).p) \
+ free((FBUF).p); \
+ if((UBUF).len > 0) { \
+ if(NULL == ((FBUF).p = malloc((UBUF).len * sizeof(int)))) \
+ ERROR; \
+ memcpy((FBUF).p, (UBUF).p, (UBUF).len * sizeof(int)); \
+ } /* end if */ \
+ else \
+ (FBUF).p = NULL; \
+ (FBUF).len = (UBUF).len; \
+} while(0)
+
+#define UPDATE_FBUF_VLS(FBUF, UBUF) \
+do { \
+ if(FBUF) \
+ free(FBUF); \
+ if(UBUF) { \
+ if(NULL == ((FBUF) = strdup(UBUF))) \
+ ERROR; \
+ } /* end if */ \
+ else \
+ (FBUF) = NULL; \
+} while(0)
+
int main(int argc, char *argv[]) {
uuid_t pool_uuid;
char *pool_grp = NULL;
hid_t file = -1, dset_vla = -1, dset_vls = -1, attr_vla = -1, attr_vls = -1, space = -1, fapl = -1;
hid_t type_vla = -1, type_vls = -1;
+ hid_t space_contig = -1, space_ncontig = -1;
hsize_t dims[2] = {4, 2};
+ hsize_t start[2], count[2];
int static_buf_vla[4][2][8];
char static_buf_vls[4][2][8];
hvl_t rbuf_vla[4][2];
char *rbuf_vls[4][2];
hvl_t wbuf_vla[4][2];
char *wbuf_vls[4][2];
+ hvl_t ebuf_vla[4][2];
+ char *ebuf_vls[4][2];
+ hvl_t fbuf_vla[4][2];
+ char *fbuf_vls[4][2];
int bogus_int = -1;
char bogus_str[3] = {'-', '1', '\0'};
const hvl_t rbuf_vla_init[4][2] = {{{1, &bogus_int}, {1, &bogus_int}},
@@ -112,6 +144,9 @@ int main(int argc, char *argv[]) {
{&static_buf_vls[3][0][0], &static_buf_vls[3][1][0]}};
int i, j, k, l;
+ memset(fbuf_vla[0], 0, sizeof(fbuf_vla));
+ memset(fbuf_vls[0], 0, sizeof(fbuf_vls));
+
(void)MPI_Init(&argc, &argv);
/* Seed random number generator */
@@ -147,6 +182,26 @@ int main(int argc, char *argv[]) {
if((space = H5Screate_simple(2, dims, NULL)) < 0)
ERROR;
+ /* Set up contiguous dataspace selection */
+ if((space_contig = H5Screate_simple(2, dims, NULL)) < 0)
+ ERROR;
+ start[0] = 1;
+ start[1] = 0;
+ count[0] = 2;
+ count[1] = 2;
+ if(H5Sselect_hyperslab(space_contig, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
+ ERROR;
+
+ /* Set up con-contiguous dataspace selection */
+ if((space_ncontig = H5Screate_simple(2, dims, NULL)) < 0)
+ ERROR;
+ start[0] = 0;
+ start[1] = 1;
+ count[0] = 4;
+ count[1] = 1;
+ if(H5Sselect_hyperslab(space_ncontig, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
+ ERROR;
+
/*
* Set up types
*/
@@ -294,6 +349,368 @@ int main(int argc, char *argv[]) {
if(verbose_g)
printf("\n");
+
+ /*
+ * Variable length array dataset test
+ */
+ /* Fill write data buffer */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ for(l = 0; l < (sizeof(static_buf_vla[0][0]) / sizeof(static_buf_vla[0][0][0])); l++)
+ static_buf_vla[j][k][l] = rand() % 100;
+
+ /* Fill write buffer */
+ memcpy(wbuf_vla, wbuf_vla_init, sizeof(wbuf_vla));
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++) {
+ /* 10% chance of having a NULL sequence */
+ if(rand() % 10)
+ wbuf_vla[j][k].len = rand() % 9;
+ else {
+ wbuf_vla[j][k].len = 0;
+ wbuf_vla[j][k].p = NULL;
+ } /* end if */
+ } /* end for */
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Writing dataset with variable length arrays. buf =\n");
+ print_vla(wbuf_vla[0]);
+ } /* end if */
+
+ /* Write data */
+ if(H5Dwrite(dset_vla, type_vla, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf_vla) < 0)
+ ERROR;
+
+ /* Read data */
+ memcpy(rbuf_vla, rbuf_vla_init, sizeof(rbuf_vla));
+ if(H5Dread(dset_vla, type_vla, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf_vla) < 0)
+ ERROR;
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Read dataset with variable length arrays. buf =\n");
+ print_vla(rbuf_vla[0]);
+ } /* end if */
+
+ /* Check buffer */
+ if(check_vla(wbuf_vla[0], rbuf_vla[0]) < 0) {
+ (void)H5Dvlen_reclaim(type_vla, space, H5P_DEFAULT, rbuf_vla);
+ ERROR;
+ } /* end if */
+
+ /* Reclaim read buffer */
+ if(H5Dvlen_reclaim(type_vla, space, H5P_DEFAULT, rbuf_vla) < 0)
+ ERROR;
+
+ if(verbose_g)
+ printf("\n");
+
+ /*
+ * Variable length string dataset test
+ */
+ /* Fill write data buffer */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ for(l = 0; l < (sizeof(static_buf_vls[0][0]) / sizeof(static_buf_vls[0][0][0])); l++)
+ static_buf_vls[j][k][l] = 'a' + rand() % 26;
+
+ /* Fill write buffer */
+ memcpy(wbuf_vls, wbuf_vls_init, sizeof(wbuf_vls));
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++) {
+ /* 10% chance of having a NULL sequence */
+ if(rand() % 10)
+ static_buf_vls[j][k][rand() % 8] = '\0';
+ else
+ wbuf_vls[j][k] = NULL;
+ } /* end for */
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Writing dataset with variable length strings. buf =\n");
+ print_vls(wbuf_vls[0]);
+ } /* end if */
+
+ /* Write data */
+ if(H5Dwrite(dset_vls, type_vls, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf_vls) < 0)
+ ERROR;
+
+ /* Read data */
+ memcpy(rbuf_vls, rbuf_vls_init, sizeof(rbuf_vls));
+ if(H5Dread(dset_vls, type_vls, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf_vls) < 0)
+ ERROR;
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Read dataset with variable length strings. buf =\n");
+ print_vls(rbuf_vls[0]);
+ } /* end if */
+
+ /* Check buffer */
+ if(check_vls(wbuf_vls[0], rbuf_vls[0]) < 0) {
+ (void)H5Dvlen_reclaim(type_vls, space, H5P_DEFAULT, rbuf_vls);
+ ERROR;
+ } /* end if */
+
+ /* Reclaim read buffer */
+ if(H5Dvlen_reclaim(type_vls, space, H5P_DEFAULT, rbuf_vls) < 0)
+ ERROR;
+
+ if(verbose_g)
+ printf("\n");
+
+ /*
+ * Variable length array dataset test with selections
+ */
+ /* Fill write data buffer */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ for(l = 0; l < (sizeof(static_buf_vla[0][0]) / sizeof(static_buf_vla[0][0][0])); l++)
+ static_buf_vla[j][k][l] = rand() % 100;
+
+ /* Fill write buffer */
+ memcpy(wbuf_vla, wbuf_vla_init, sizeof(wbuf_vla));
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++) {
+ /* 10% chance of having a NULL sequence */
+ if(rand() % 10)
+ wbuf_vla[j][k].len = rand() % 9;
+ else {
+ wbuf_vla[j][k].len = 0;
+ wbuf_vla[j][k].p = NULL;
+ } /* end if */
+ } /* end for */
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Writing dataset with variable length arrays. buf =\n");
+ print_vla(wbuf_vla[0]);
+ } /* end if */
+
+ /* Write data to whole dataset */
+ if(H5Dwrite(dset_vla, type_vla, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf_vla) < 0)
+ ERROR;
+
+ /* Update fbuf */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ UPDATE_FBUF_VLA(fbuf_vla[j][k], wbuf_vla[j][k]);
+
+ /* Read data from non-contiguous to contiguous */
+ memcpy(rbuf_vla, rbuf_vla_init, sizeof(rbuf_vla));
+ memcpy(ebuf_vla, rbuf_vla_init, sizeof(rbuf_vla));
+ if(H5Dread(dset_vla, type_vla, space_contig, space_ncontig, H5P_DEFAULT, rbuf_vla) < 0)
+ ERROR;
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Read dataset with variable length arrays with selection. buf =\n");
+ print_vla(rbuf_vla[0]);
+ } /* end if */
+
+ /* Update ebuf */
+ ebuf_vla[1][0] = fbuf_vla[0][1];
+ ebuf_vla[1][1] = fbuf_vla[1][1];
+ ebuf_vla[2][0] = fbuf_vla[2][1];
+ ebuf_vla[2][1] = fbuf_vla[3][1];
+
+ /* Check buffer */
+ if(check_vla(ebuf_vla[0], rbuf_vla[0]) < 0) {
+ (void)H5Dvlen_reclaim(type_vla, space_contig, H5P_DEFAULT, rbuf_vla);
+ ERROR;
+ } /* end if */
+
+ /* Reclaim read buffer */
+ if(H5Dvlen_reclaim(type_vla, space_contig, H5P_DEFAULT, rbuf_vla) < 0)
+ ERROR;
+
+ if(verbose_g)
+ printf("\n");
+
+ /* Fill write data buffer */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ for(l = 0; l < (sizeof(static_buf_vla[0][0]) / sizeof(static_buf_vla[0][0][0])); l++)
+ static_buf_vla[j][k][l] = rand() % 100;
+
+ /* Fill write buffer */
+ memcpy(wbuf_vla, wbuf_vla_init, sizeof(wbuf_vla));
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++) {
+ /* 10% chance of having a NULL sequence */
+ if(rand() % 10)
+ wbuf_vla[j][k].len = rand() % 9;
+ else {
+ wbuf_vla[j][k].len = 0;
+ wbuf_vla[j][k].p = NULL;
+ } /* end if */
+ } /* end for */
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Writing dataset with variable length arrays with selection. buf =\n");
+ print_vla(wbuf_vla[0]);
+ } /* end if */
+
+ /* Write data to dataset from non-contiguous to contiguous */
+ if(H5Dwrite(dset_vla, type_vla, space_ncontig, space_contig, H5P_DEFAULT, wbuf_vla) < 0)
+ ERROR;
+
+ /* Update fbuf */
+ UPDATE_FBUF_VLA(fbuf_vla[1][0], wbuf_vla[0][1]);
+ UPDATE_FBUF_VLA(fbuf_vla[1][1], wbuf_vla[1][1]);
+ UPDATE_FBUF_VLA(fbuf_vla[2][0], wbuf_vla[2][1]);
+ UPDATE_FBUF_VLA(fbuf_vla[2][1], wbuf_vla[3][1]);
+
+ /* Read entire dataset */
+ memcpy(rbuf_vla, rbuf_vla_init, sizeof(rbuf_vla));
+ memcpy(ebuf_vla, rbuf_vla_init, sizeof(rbuf_vla));
+ if(H5Dread(dset_vla, type_vla, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf_vla) < 0)
+ ERROR;
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Read dataset with variable length arrays. buf =\n");
+ print_vla(rbuf_vla[0]);
+ } /* end if */
+
+ /* Check buffer */
+ if(check_vla(fbuf_vla[0], rbuf_vla[0]) < 0) {
+ (void)H5Dvlen_reclaim(type_vla, space, H5P_DEFAULT, rbuf_vla);
+ ERROR;
+ } /* end if */
+
+ /* Reclaim read buffer */
+ if(H5Dvlen_reclaim(type_vla, space, H5P_DEFAULT, rbuf_vla) < 0)
+ ERROR;
+
+ if(verbose_g)
+ printf("\n");
+
+ /*
+ * Variable length string dataset test with selections
+ */
+ /* Fill write data buffer */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ for(l = 0; l < (sizeof(static_buf_vls[0][0]) / sizeof(static_buf_vls[0][0][0])); l++)
+ static_buf_vls[j][k][l] = 'a' + rand() % 26;
+
+ /* Fill write buffer */
+ memcpy(wbuf_vls, wbuf_vls_init, sizeof(wbuf_vls));
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++) {
+ /* 10% chance of having a NULL sequence */
+ if(rand() % 10)
+ static_buf_vls[j][k][rand() % 8] = '\0';
+ else
+ wbuf_vls[j][k] = NULL;
+ } /* end for */
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Writing dataset with variable length strings. buf =\n");
+ print_vls(wbuf_vls[0]);
+ } /* end if */
+
+ /* Write data to whole dataset */
+ if(H5Dwrite(dset_vls, type_vls, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf_vls) < 0)
+ ERROR;
+
+ /* Update fbuf */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ UPDATE_FBUF_VLS(fbuf_vls[j][k], wbuf_vls[j][k]);
+
+ /* Read data from contiguous to non-contiguous */
+ memcpy(rbuf_vls, rbuf_vls_init, sizeof(rbuf_vls));
+ memcpy(ebuf_vls, rbuf_vls_init, sizeof(rbuf_vls));
+ if(H5Dread(dset_vls, type_vls, space_ncontig, space_contig, H5P_DEFAULT, rbuf_vls) < 0)
+ ERROR;
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Read dataset with variable length strings with selection. buf =\n");
+ print_vls(rbuf_vls[0]);
+ } /* end if */
+
+ /* Update ebuf */
+ ebuf_vls[0][1] = fbuf_vls[1][0];
+ ebuf_vls[1][1] = fbuf_vls[1][1];
+ ebuf_vls[2][1] = fbuf_vls[2][0];
+ ebuf_vls[3][1] = fbuf_vls[2][1];
+
+ /* Check buffer */
+ if(check_vls(ebuf_vls[0], rbuf_vls[0]) < 0) {
+ (void)H5Dvlen_reclaim(type_vls, space_ncontig, H5P_DEFAULT, rbuf_vls);
+ ERROR;
+ } /* end if */
+
+ /* Reclaim read buffer */
+ if(H5Dvlen_reclaim(type_vls, space_ncontig, H5P_DEFAULT, rbuf_vls) < 0)
+ ERROR;
+
+ if(verbose_g)
+ printf("\n");
+
+ /* Fill write data buffer */
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++)
+ for(l = 0; l < (sizeof(static_buf_vls[0][0]) / sizeof(static_buf_vls[0][0][0])); l++)
+ static_buf_vls[j][k][l] = 'a' + rand() % 26;
+
+ /* Fill write buffer */
+ memcpy(wbuf_vls, wbuf_vls_init, sizeof(wbuf_vls));
+ for(j = 0; j < dims[0]; j++)
+ for(k = 0; k < dims[1]; k++) {
+ /* 10% chance of having a NULL sequence */
+ if(rand() % 10)
+ static_buf_vls[j][k][rand() % 8] = '\0';
+ else
+ wbuf_vls[j][k] = NULL;
+ } /* end for */
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Writing dataset with variable length strings with selection. buf =\n");
+ print_vls(wbuf_vls[0]);
+ } /* end if */
+
+ /* Write data to dataset from non-contiguous to contiguous */
+ if(H5Dwrite(dset_vls, type_vls, space_contig, space_ncontig, H5P_DEFAULT, wbuf_vls) < 0)
+ ERROR;
+
+ /* Update fbuf */
+ UPDATE_FBUF_VLS(fbuf_vls[0][1], wbuf_vls[1][0]);
+ UPDATE_FBUF_VLS(fbuf_vls[1][1], wbuf_vls[1][1]);
+ UPDATE_FBUF_VLS(fbuf_vls[2][1], wbuf_vls[2][0]);
+ UPDATE_FBUF_VLS(fbuf_vls[3][1], wbuf_vls[2][1]);
+
+ /* Read entire dataset */
+ memcpy(rbuf_vls, rbuf_vls_init, sizeof(rbuf_vls));
+ memcpy(ebuf_vls, rbuf_vls_init, sizeof(rbuf_vls));
+ if(H5Dread(dset_vls, type_vls, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf_vls) < 0)
+ ERROR;
+
+ /* Print message */
+ if(verbose_g) {
+ printf("Read dataset with variable length strings. buf =\n");
+ print_vls(rbuf_vls[0]);
+ } /* end if */
+
+ /* Check buffer */
+ if(check_vls(fbuf_vls[0], rbuf_vls[0]) < 0) {
+ (void)H5Dvlen_reclaim(type_vls, space, H5P_DEFAULT, rbuf_vls);
+ ERROR;
+ } /* end if */
+
+ /* Reclaim read buffer */
+ if(H5Dvlen_reclaim(type_vls, space, H5P_DEFAULT, rbuf_vls) < 0)
+ ERROR;
+
+ if(verbose_g)
+ printf("\n");
} /* end for */
/*
@@ -315,9 +732,21 @@ int main(int argc, char *argv[]) {
ERROR;
if(H5Sclose(space) < 0)
ERROR;
+ if(H5Sclose(space_contig) < 0)
+ ERROR;
+ if(H5Sclose(space_ncontig) < 0)
+ ERROR;
if(H5Pclose(fapl) < 0)
ERROR;
+ for(i = 0; i < dims[0]; i++)
+ for(j = 0; j < dims[1]; j++) {
+ if(fbuf_vla[i][j].p)
+ free(fbuf_vla[i][j].p);
+ if(fbuf_vls[i][j])
+ free(fbuf_vls[i][j]);
+ } /* end for */
+
printf("Success\n");
(void)MPI_Finalize();
@@ -333,9 +762,19 @@ error:
H5Tclose(type_vla);
H5Tclose(type_vls);
H5Sclose(space);
+ H5Sclose(space_contig);
+ H5Sclose(space_ncontig);
H5Pclose(fapl);
} H5E_END_TRY;
+ for(i = 0; i < dims[0]; i++)
+ for(j = 0; j < dims[1]; j++) {
+ if(fbuf_vla[i][j].p)
+ free(fbuf_vla[i][j].p);
+ if(fbuf_vls[i][j])
+ free(fbuf_vls[i][j]);
+ } /* end for */
+
(void)MPI_Finalize();
return 1;
}
diff --git a/src/H5VLdaosm.c b/src/H5VLdaosm.c
index 12ec6b3..434a329 100644
--- a/src/H5VLdaosm.c
+++ b/src/H5VLdaosm.c
@@ -96,6 +96,24 @@ typedef struct H5VL_daosm_scatter_cb_ud_t {
size_t len;
} H5VL_daosm_scatter_cb_ud_t;
+/* Udata type for memory space H5Diterate callback */
+typedef struct {
+ daos_iod_t *iods;
+ daos_sg_list_t *sgls;
+ daos_iov_t *sg_iovs;
+ hbool_t is_vl_str;
+ size_t base_type_size;
+ uint64_t offset;
+ uint64_t idx;
+} H5VL_daosm_vl_mem_ud_t;
+
+/* Udata type for file space H5Diterate callback */
+typedef struct {
+ uint8_t **akeys;
+ daos_iod_t *iods;
+ uint64_t idx;
+} H5VL_daosm_vl_file_ud_t;
+
/* Prototypes */
static void *H5VL_daosm_fapl_copy(const void *_old_fa);
static herr_t H5VL_daosm_fapl_free(void *_fa);
@@ -211,6 +229,12 @@ static herr_t H5VL_daosm_sel_to_recx_iov(H5S_t *space, size_t type_size,
void *buf, daos_recx_t **recxs, daos_iov_t **sg_iovs, size_t *list_nused);
static herr_t H5VL_daosm_scatter_cb(const void **src_buf,
size_t *src_buf_bytes_used, void *_udata);
+static herr_t H5VL_daosm_dataset_mem_vl_rd_cb(void *_elem, hid_t type_id,
+ unsigned ndim, const hsize_t *point, void *_udata);
+static herr_t H5VL_daosm_dataset_file_vl_cb(void *_elem, hid_t type_id,
+ unsigned ndim, const hsize_t *point, void *_udata);
+static herr_t H5VL_daosm_dataset_mem_vl_wr_cb(void *_elem, hid_t type_id,
+ unsigned ndim, const hsize_t *point, void *_udata);
static herr_t H5VL_daosm_object_close(void *_obj, hid_t dxpl_id, void **req);
@@ -4066,6 +4090,146 @@ H5VL_daosm_scatter_cb(const void **src_buf, size_t *src_buf_bytes_used,
/*-------------------------------------------------------------------------
+ * Function: H5VL_daosm_dataset_mem_vl_rd_cb
+ *
+ * Purpose: H5Diterate callback for iterating over the memory space
+ * before reading vl data. Allocates vl read buffers,
+ * up scatter gather lists (sgls), and reshapes iods if
+ * necessary to skip empty elements.
+ *
+ * Return: Success: 0
+ * Failure: -1, dataset not written.
+ *
+ * Programmer: Neil Fortner
+ * May, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5VL_daosm_dataset_mem_vl_rd_cb(void *_elem, hid_t H5_ATTR_UNUSED type_id,
+ unsigned H5_ATTR_UNUSED ndim, const hsize_t H5_ATTR_UNUSED *point,
+ void *_udata)
+{
+ H5VL_daosm_vl_mem_ud_t *udata = (H5VL_daosm_vl_mem_ud_t *)_udata;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Set up constant sgl info */
+ udata->sgls[udata->idx].sg_nr.num = 1;
+ udata->sgls[udata->idx].sg_iovs = &udata->sg_iovs[udata->idx];
+
+ /* Check for empty element */
+ if(udata->iods[udata->idx].iod_size == 0) {
+ /* Increment offset, slide down following elements */
+ udata->offset++;
+
+ /* Zero out read buffer */
+ if(udata->is_vl_str)
+ *(char **)_elem = NULL;
+ else
+ HDmemset(_elem, 0, sizeof(hvl_t));
+ } /* end if */
+ else {
+ HDassert(udata->idx >= udata->offset);
+
+ /* Check for vlen string */
+ if(udata->is_vl_str) {
+ char *elem = NULL;
+
+ /* Allocate buffer for this vl element */
+ if(NULL == (elem = (char *)HDmalloc((size_t)udata->iods[udata->idx].iod_size + 1)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate vl data buffer")
+ *(char **)_elem = elem;
+
+ /* Add null terminator */
+ elem[udata->iods[udata->idx].iod_size] = '\0';
+
+ /* Set buffer location in sgl */
+ daos_iov_set(&udata->sg_iovs[udata->idx - udata->offset], elem, udata->iods[udata->idx].iod_size);
+ } /* end if */
+ else {
+ /* Standard vlen, find hvl_t struct for this element */
+ hvl_t *elem = (hvl_t *)_elem;
+
+ HDassert(udata->base_type_size > 0);
+
+ /* Allocate buffer for this vl element and set size */
+ elem->len = (size_t)udata->iods[udata->idx].iod_size / udata->base_type_size;
+ if(NULL == (elem->p = HDmalloc((size_t)udata->iods[udata->idx].iod_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate vl data buffer")
+
+ /* Set buffer location in sgl */
+ daos_iov_set(&udata->sg_iovs[udata->idx - udata->offset], elem->p, udata->iods[udata->idx].iod_size);
+ } /* end if */
+
+ /* Slide down iod if necessary */
+ if(udata->offset)
+ udata->iods[udata->idx - udata->offset] = udata->iods[udata->idx];
+ } /* end else */
+
+ /* Advance idx */
+ udata->idx++;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5VL_daosm_dataset_mem_vl_rd_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5VL_daosm_dataset_file_vl_cb
+ *
+ * Purpose: H5Diterate callback for iterating over the file space
+ * before vl data I/O. Sets up akeys and iods (except for
+ * iod record sizes).
+ *
+ * Return: Success: 0
+ * Failure: -1, dataset not written.
+ *
+ * Programmer: Neil Fortner
+ * May, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5VL_daosm_dataset_file_vl_cb(void H5_ATTR_UNUSED *_elem,
+ hid_t H5_ATTR_UNUSED type_id, unsigned ndim, const hsize_t *point,
+ void *_udata)
+{
+ H5VL_daosm_vl_file_ud_t *udata = (H5VL_daosm_vl_file_ud_t *)_udata;
+ size_t akey_len = ndim * sizeof(uint64_t);
+ uint64_t coordu64;
+ uint8_t *p;
+ unsigned i;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Create akey for this element */
+ if(NULL == (udata->akeys[udata->idx] = (uint8_t *)H5MM_malloc(akey_len)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for akey")
+ p = udata->akeys[udata->idx];
+ for(i = 0; i < ndim; i++) {
+ coordu64 = (uint64_t)point[i];
+ UINT64ENCODE(p, coordu64)
+ } /* end for */
+
+ /* Set up iod, size was set in memory callback or initialized in main read
+ * function. Use "single" records of varying size. */
+ daos_iov_set(&udata->iods[udata->idx].iod_name, (void *)udata->akeys[udata->idx], (daos_size_t)akey_len);
+ daos_csum_set(&udata->iods[udata->idx].iod_kcsum, NULL, 0);
+ udata->iods[udata->idx].iod_nr = 1u;
+ udata->iods[udata->idx].iod_type = DAOS_IOD_SINGLE;
+
+ /* Advance idx */
+ udata->idx++;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5VL_daosm_dataset_file_vl_cb() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5VL_daosm_dataset_read
*
* Purpose: Reads raw data from a dataset into a buffer.
@@ -4089,26 +4253,28 @@ H5VL_daosm_dataset_read(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
hsize_t dim[H5S_MAX_RANK];
hid_t real_file_space_id;
hid_t real_mem_space_id;
- H5S_t *space = NULL;
+ hssize_t num_elem = -1;
uint64_t chunk_coords[H5S_MAX_RANK];
daos_key_t dkey;
- daos_iod_t iod;
+ uint8_t **akeys = NULL;
+ daos_iod_t *iods = NULL;
+ daos_sg_list_t *sgls = NULL;
daos_recx_t recx;
daos_recx_t *recxs = &recx;
- daos_sg_list_t sgl;
daos_iov_t sg_iov;
daos_iov_t *sg_iovs = &sg_iov;
- size_t tot_nseq;
uint8_t dkey_buf[1 + H5S_MAX_RANK];
- uint8_t akey = H5VL_DAOSM_CHUNK_KEY;
- size_t file_type_size;
- htri_t types_equal;
+ hid_t base_type_id = FAIL;
+ size_t base_type_size = 0;
void *tconv_buf = NULL;
void *bkg_buf = NULL;
+ H5T_class_t type_class;
+ hbool_t is_vl = FALSE;
+ htri_t is_vl_str = FALSE;
H5VL_daosm_tconv_reuse_t reuse = H5VL_DAOSM_TCONV_REUSE_NONE;
uint8_t *p;
int ret;
- int i;
+ uint64_t i;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
@@ -4129,156 +4295,251 @@ H5VL_daosm_dataset_read(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
else
real_mem_space_id = mem_space_id;
- /* Get datatype size */
- if((file_type_size = H5Tget_size(dset->type_id)) == 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype size")
-
/* Encode dkey (chunk coordinates). Prefix with '\0' to avoid accidental
* collisions with other d-keys in this object. For now just 1 chunk,
* starting at 0. */
HDmemset(chunk_coords, 0, sizeof(chunk_coords)); //DSMINC
p = dkey_buf;
*p++ = (uint8_t)'\0';
- for(i = 0; i < ndims; i++)
+ for(i = 0; i < (uint64_t)ndims; i++)
UINT64ENCODE(p, chunk_coords[i])
/* Set up dkey */
daos_iov_set(&dkey, dkey_buf, (daos_size_t)(1 + ((size_t)ndims * sizeof(chunk_coords[0]))));
- /* Set up iod */
- HDmemset(&iod, 0, sizeof(iod));
- daos_iov_set(&iod.iod_name, (void *)&akey, (daos_size_t)(sizeof(akey)));
- daos_csum_set(&iod.iod_kcsum, NULL, 0);
- iod.iod_size = file_type_size;
- iod.iod_type = DAOS_IOD_ARRAY;
+ /* Check for vlen */
+ if(H5T_NO_CLASS == (type_class = H5Tget_class(mem_type_id)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype class")
+ if(type_class == H5T_VLEN) {
+ is_vl = TRUE;
- /* Get file dataspace object */
- if(NULL == (space = (H5S_t *)H5I_object(real_file_space_id)))
- HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+ /* Calculate base type size */
+ if((base_type_id = H5Tget_super(mem_type_id)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype base type")
+ if(0 == (base_type_size = H5Tget_size(base_type_id)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype base type size")
+ } /* end if */
+ else if(type_class == H5T_STRING) {
+ /* check for vlen string */
+ if((is_vl_str = H5Tis_variable_str(mem_type_id)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check for variable length string")
+ if(is_vl_str)
+ is_vl = TRUE;
+ } /* end if */
- /* Check if the types are equal */
- if((types_equal = H5Tequal(dset->type_id, mem_type_id)) < 0)
- HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOMPARE, FAIL, "can't check if types are equal")
- if(types_equal) {
- /* No type conversion necessary */
- /* Check for memory space is H5S_ALL, use file space in this case */
- if(mem_space_id == H5S_ALL) {
- /* Calculate both recxs and sg_iovs at the same time from file space */
- if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, &recxs, &sg_iovs, &tot_nseq) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
- iod.iod_nr = (unsigned)tot_nseq;
- sgl.sg_nr.num = (uint32_t)tot_nseq;
- } /* end if */
- else {
- /* Calculate recxs from file space */
- if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, &recxs, NULL, &tot_nseq) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
- iod.iod_nr = (unsigned)tot_nseq;
+ /* Check for variable length */
+ if(is_vl) {
+ H5VL_daosm_vl_mem_ud_t mem_ud;
+ H5VL_daosm_vl_file_ud_t file_ud;
- /* Get memory dataspace object */
- if(NULL == (space = (H5S_t *)H5I_object(real_mem_space_id)))
- HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+ /* Get number of elements in selection */
+ if((num_elem = H5Sget_select_npoints(real_mem_space_id)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get number of points in selection")
- /* Calculate sg_iovs from mem space */
- if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, NULL, &sg_iovs, &tot_nseq) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
- sgl.sg_nr.num = (uint32_t)tot_nseq;
- } /* end else */
+ /* Allocate array of akey pointers */
+ if(NULL == (akeys = (uint8_t **)H5MM_calloc((size_t)num_elem * sizeof(uint8_t *))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for akey array")
+
+ /* Allocate array of iods */
+ if(NULL == (iods = (daos_iod_t *)H5MM_calloc((size_t)num_elem * sizeof(daos_iod_t))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for I/O descriptor array")
+
+ /* Fill in size fields of iod as DAOS_REC_ANY so we can read the vl
+ * sizes */
+ for(i = 0; i < (uint64_t)num_elem; i++)
+ iods[i].iod_size = DAOS_REC_ANY;
+
+ /* Iterate over file selection. Note the bogus buffer and type_id,
+ * these don't matter since the "elem" parameter of the callback is not
+ * used. */
+ file_ud.akeys = akeys;
+ file_ud.iods = iods;
+ file_ud.idx = 0;
+ if(H5Diterate((void *)buf, mem_type_id, real_file_space_id, H5VL_daosm_dataset_file_vl_cb, &file_ud) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "file selection iteration failed")
+ HDassert(file_ud.idx == (uint64_t)num_elem);
+
+ /* Read vl sizes from dataset */
+ /* Note cast to unsigned reduces width to 32 bits. Should eventually
+ * check for overflow and iterate over 2^32 size blocks */
+ if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, (unsigned)num_elem, iods, NULL, NULL /*maps*/, NULL /*event*/)))
+ HGOTO_ERROR(H5E_ATTR, H5E_READERROR, FAIL, "can't read vl data sizes from dataset: %d", ret)
- /* Point iod and sgl to lists generated above */
- iod.iod_recxs = recxs;
- sgl.sg_iovs = sg_iovs;
+ /* Allocate array of sg_iovs */
+ if(NULL == (sg_iovs = (daos_iov_t *)H5MM_malloc((size_t)num_elem * sizeof(daos_iov_t))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for scatter gather list")
+
+ /* Allocate array of sgls */
+ if(NULL == (sgls = (daos_sg_list_t *)H5MM_malloc((size_t)num_elem * sizeof(daos_sg_list_t))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for scatter gather list array")
+
+ /* Iterate over memory selection */
+ mem_ud.iods = iods;
+ mem_ud.sgls = sgls;
+ mem_ud.sg_iovs = sg_iovs;
+ mem_ud.is_vl_str = is_vl_str;
+ mem_ud.base_type_size = base_type_size;
+ mem_ud.offset = 0;
+ mem_ud.idx = 0;
+ if(H5Diterate((void *)buf, mem_type_id, real_mem_space_id, H5VL_daosm_dataset_mem_vl_rd_cb, &mem_ud) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "memory selection iteration failed")
+ HDassert(mem_ud.idx == (uint64_t)num_elem);
/* Read data from dataset */
- if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*maps*/, NULL /*event*/)))
+ /* Note cast to unsigned reduces width to 32 bits. Should eventually
+ * check for overflow and iterate over 2^32 size blocks */
+ if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, (unsigned)(num_elem - mem_ud.offset), iods, sgls, NULL /*maps*/, NULL /*event*/)))
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data from dataset: %d", ret)
} /* end if */
else {
- hssize_t num_elem;
- size_t nseq_tmp;
- size_t nelem_tmp;
- hsize_t sel_off;
- size_t sel_len;
- size_t mem_type_size;
- hbool_t fill_bkg = FALSE;
- hbool_t contig;
-
- /* Type conversion necessary */
- /* Get number of elements in selection */
- if((num_elem = H5Sget_select_npoints(real_mem_space_id)) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get number of points in selection")
+ H5S_t *space = NULL;
+ daos_iod_t iod;
+ daos_sg_list_t sgl;
+ uint8_t akey = H5VL_DAOSM_CHUNK_KEY;
+ size_t tot_nseq;
+ size_t file_type_size;
+ htri_t types_equal;
- /* Calculate recxs from file space */
- if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, &recxs, NULL, &tot_nseq) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
- iod.iod_nr = (unsigned)tot_nseq;
- iod.iod_recxs = recxs;
+ /* Get datatype size */
+ if((file_type_size = H5Tget_size(dset->type_id)) == 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype size")
- /* Set up constant sgl info */
- sgl.sg_nr.num = 1;
- sgl.sg_iovs = &sg_iov;
+ /* Set up iod */
+ HDmemset(&iod, 0, sizeof(iod));
+ daos_iov_set(&iod.iod_name, (void *)&akey, (daos_size_t)(sizeof(akey)));
+ daos_csum_set(&iod.iod_kcsum, NULL, 0);
+ iod.iod_size = file_type_size;
+ iod.iod_type = DAOS_IOD_ARRAY;
- /* Check for contiguous memory buffer */
- /* Get memory dataspace object */
- if(NULL == (space = (H5S_t *)H5I_object(real_mem_space_id)))
+ /* Get file dataspace object */
+ if(NULL == (space = (H5S_t *)H5I_object(real_file_space_id)))
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
- /* Initialize selection iterator */
- if(H5S_select_iter_init(&sel_iter, space, (size_t)1) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator")
- sel_iter_init = TRUE; /* Selection iteration info has been initialized */
+ /* Check if the types are equal */
+ if((types_equal = H5Tequal(dset->type_id, mem_type_id)) < 0)
+ HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOMPARE, FAIL, "can't check if types are equal")
+ if(types_equal) {
+ /* No type conversion necessary */
+ /* Check for memory space is H5S_ALL, use file space in this case */
+ if(mem_space_id == H5S_ALL) {
+ /* Calculate both recxs and sg_iovs at the same time from file space */
+ if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, &recxs, &sg_iovs, &tot_nseq) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
+ iod.iod_nr = (unsigned)tot_nseq;
+ sgl.sg_nr.num = (uint32_t)tot_nseq;
+ } /* end if */
+ else {
+ /* Calculate recxs from file space */
+ if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, &recxs, NULL, &tot_nseq) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
+ iod.iod_nr = (unsigned)tot_nseq;
+
+ /* Get memory dataspace object */
+ if(NULL == (space = (H5S_t *)H5I_object(real_mem_space_id)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+
+ /* Calculate sg_iovs from mem space */
+ if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, NULL, &sg_iovs, &tot_nseq) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
+ sgl.sg_nr.num = (uint32_t)tot_nseq;
+ } /* end else */
- /* Get the sequence list - only check the first sequence because we only
- * care if it is contiguous and if so where the contiguous selection
- * begins */
- if(H5S_SELECT_GET_SEQ_LIST(space, 0, &sel_iter, (size_t)1, (size_t)-1, &nseq_tmp, &nelem_tmp, &sel_off, &sel_len) < 0)
- HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed")
- contig = (sel_len == (size_t)num_elem);
+ /* Point iod and sgl to lists generated above */
+ iod.iod_recxs = recxs;
+ sgl.sg_iovs = sg_iovs;
- /* Initialize type conversion */
- if(H5VL_daosm_tconv_init(dset->type_id, &file_type_size, mem_type_id, &mem_type_size, (size_t)num_elem, &tconv_buf, &bkg_buf, contig ? &reuse : NULL, &fill_bkg) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize type conversion")
-
- /* Reuse buffer as appropriate */
- if(contig) {
- sel_off *= (hsize_t)mem_type_size;
- if(reuse == H5VL_DAOSM_TCONV_REUSE_TCONV)
- tconv_buf = (char *)buf + (size_t)sel_off;
- else if(reuse == H5VL_DAOSM_TCONV_REUSE_BKG)
- bkg_buf = (char *)buf + (size_t)sel_off;
+ /* Read data from dataset */
+ if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*maps*/, NULL /*event*/)))
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data from dataset: %d", ret)
} /* end if */
+ else {
+ size_t nseq_tmp;
+ size_t nelem_tmp;
+ hsize_t sel_off;
+ size_t sel_len;
+ size_t mem_type_size;
+ hbool_t fill_bkg = FALSE;
+ hbool_t contig;
+
+ /* Type conversion necessary */
+ /* Get number of elements in selection */
+ if((num_elem = H5Sget_select_npoints(real_mem_space_id)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get number of points in selection")
- /* Set sg_iov to point to tconv_buf */
- daos_iov_set(&sg_iov, tconv_buf, (daos_size_t)num_elem * (daos_size_t)file_type_size);
+ /* Calculate recxs from file space */
+ if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, buf, &recxs, NULL, &tot_nseq) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
+ iod.iod_nr = (unsigned)tot_nseq;
+ iod.iod_recxs = recxs;
- /* Read data to tconv_buf */
- if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*maps*/, NULL /*event*/)))
- HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data from attribute: %d", ret)
+ /* Set up constant sgl info */
+ sgl.sg_nr.num = 1;
+ sgl.sg_iovs = &sg_iov;
- /* Gather data to background buffer if necessary */
- if(fill_bkg && (reuse != H5VL_DAOSM_TCONV_REUSE_BKG))
- if(H5Dgather(real_mem_space_id, buf, mem_type_id, (size_t)num_elem * mem_type_size, bkg_buf, NULL, NULL) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't gather data to background buffer")
+ /* Check for contiguous memory buffer */
+ /* Get memory dataspace object */
+ if(NULL == (space = (H5S_t *)H5I_object(real_mem_space_id)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
- /* Perform type conversion */
- if(H5Tconvert(dset->type_id, mem_type_id, (size_t)num_elem, tconv_buf, bkg_buf, dxpl_id) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't perform type conversion")
+ /* Initialize selection iterator */
+ if(H5S_select_iter_init(&sel_iter, space, (size_t)1) < 0)
+ HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator")
+ sel_iter_init = TRUE; /* Selection iteration info has been initialized */
+
+ /* Get the sequence list - only check the first sequence because we only
+ * care if it is contiguous and if so where the contiguous selection
+ * begins */
+ if(H5S_SELECT_GET_SEQ_LIST(space, 0, &sel_iter, (size_t)1, (size_t)-1, &nseq_tmp, &nelem_tmp, &sel_off, &sel_len) < 0)
+ HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed")
+ contig = (sel_len == (size_t)num_elem);
+
+ /* Initialize type conversion */
+ if(H5VL_daosm_tconv_init(dset->type_id, &file_type_size, mem_type_id, &mem_type_size, (size_t)num_elem, &tconv_buf, &bkg_buf, contig ? &reuse : NULL, &fill_bkg) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize type conversion")
+
+ /* Reuse buffer as appropriate */
+ if(contig) {
+ sel_off *= (hsize_t)mem_type_size;
+ if(reuse == H5VL_DAOSM_TCONV_REUSE_TCONV)
+ tconv_buf = (char *)buf + (size_t)sel_off;
+ else if(reuse == H5VL_DAOSM_TCONV_REUSE_BKG)
+ bkg_buf = (char *)buf + (size_t)sel_off;
+ } /* end if */
- /* Scatter data to memory buffer if necessary */
- if(reuse != H5VL_DAOSM_TCONV_REUSE_TCONV) {
- H5VL_daosm_scatter_cb_ud_t scatter_cb_ud;
+ /* Set sg_iov to point to tconv_buf */
+ daos_iov_set(&sg_iov, tconv_buf, (daos_size_t)num_elem * (daos_size_t)file_type_size);
- scatter_cb_ud.buf = tconv_buf;
- scatter_cb_ud.len = (size_t)num_elem * mem_type_size;
- if(H5Dscatter(H5VL_daosm_scatter_cb, &scatter_cb_ud, mem_type_id, real_mem_space_id, buf) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't scatter data to read buffer")
- } /* end if */
+ /* Read data to tconv_buf */
+ if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*maps*/, NULL /*event*/)))
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data from attribute: %d", ret)
+
+ /* Gather data to background buffer if necessary */
+ if(fill_bkg && (reuse != H5VL_DAOSM_TCONV_REUSE_BKG))
+ if(H5Dgather(real_mem_space_id, buf, mem_type_id, (size_t)num_elem * mem_type_size, bkg_buf, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't gather data to background buffer")
+
+ /* Perform type conversion */
+ if(H5Tconvert(dset->type_id, mem_type_id, (size_t)num_elem, tconv_buf, bkg_buf, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't perform type conversion")
+
+ /* Scatter data to memory buffer if necessary */
+ if(reuse != H5VL_DAOSM_TCONV_REUSE_TCONV) {
+ H5VL_daosm_scatter_cb_ud_t scatter_cb_ud;
+
+ scatter_cb_ud.buf = tconv_buf;
+ scatter_cb_ud.len = (size_t)num_elem * mem_type_size;
+ if(H5Dscatter(H5VL_daosm_scatter_cb, &scatter_cb_ud, mem_type_id, real_mem_space_id, buf) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't scatter data to read buffer")
+ } /* end if */
+ } /* end else */
} /* end else */
done:
/* Free memory */
+ iods = (daos_iod_t *)H5MM_xfree(iods);
if(recxs != &recx)
H5MM_free(recxs);
+ sgls = (daos_sg_list_t *)H5MM_xfree(sgls);
if(sg_iovs != &sg_iov)
H5MM_free(sg_iovs);
if(tconv_buf && (reuse != H5VL_DAOSM_TCONV_REUSE_TCONV))
@@ -4286,6 +4547,16 @@ done:
if(bkg_buf && (reuse != H5VL_DAOSM_TCONV_REUSE_BKG))
H5MM_free(bkg_buf);
+ if(akeys) {
+ for(i = 0; i < (uint64_t)num_elem; i++)
+ H5MM_xfree(akeys[i]);
+ H5MM_free(akeys);
+ } /* end if */
+
+ if(base_type_id != FAIL)
+ if(H5I_dec_app_ref(base_type_id) < 0)
+ HDONE_ERROR(H5E_ATTR, H5E_CLOSEERROR, FAIL, "can't close base type id")
+
/* Release selection iterator */
if(sel_iter_init && H5S_SELECT_ITER_RELEASE(&sel_iter) < 0)
HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator")
@@ -4295,6 +4566,79 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5VL_daosm_dataset_mem_vl_wr_cb
+ *
+ * Purpose: H5Diterate callback for iterating over the memory space
+ * before writing vl data. Sets up scatter gather lists
+ * (sgls) and sets the record sizes in iods.
+ *
+ * Return: Success: 0
+ * Failure: -1, dataset not written.
+ *
+ * Programmer: Neil Fortner
+ * May, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5VL_daosm_dataset_mem_vl_wr_cb(void *_elem, hid_t H5_ATTR_UNUSED type_id,
+ unsigned H5_ATTR_UNUSED ndim, const hsize_t H5_ATTR_UNUSED *point,
+ void *_udata)
+{
+ H5VL_daosm_vl_mem_ud_t *udata = (H5VL_daosm_vl_mem_ud_t *)_udata;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ /* Set up constant sgl info */
+ udata->sgls[udata->idx].sg_nr.num = 1;
+ udata->sgls[udata->idx].sg_iovs = &udata->sg_iovs[udata->idx];
+
+ /* Check for vlen string */
+ if(udata->is_vl_str) {
+ /* Find string for this element */
+ char *elem = *(char **)_elem;
+
+ /* Set string length in iod and buffer location in sgl. If we are
+ * writing an empty string ("\0"), increase the size by one to
+ * differentiate it from NULL strings. Note that this will cause the
+ * read buffer to be one byte longer than it needs to be in this case.
+ * This should not cause any ill effects. */
+ if(elem) {
+ udata->iods[udata->idx].iod_size = (daos_size_t)HDstrlen(elem);
+ if(udata->iods[udata->idx].iod_size == 0)
+ udata->iods[udata->idx].iod_size = 1;
+ daos_iov_set(&udata->sg_iovs[udata->idx], (void *)elem, udata->iods[udata->idx].iod_size);
+ } /* end if */
+ else {
+ udata->iods[udata->idx].iod_size = 0;
+ daos_iov_set(&udata->sg_iovs[udata->idx], NULL, 0);
+ } /* end else */
+ } /* end if */
+ else {
+ /* Standard vlen, find hvl_t struct for this element */
+ hvl_t *elem = (hvl_t *)_elem;
+
+ HDassert(udata->base_type_size > 0);
+
+ /* Set buffer length in iod and buffer location in sgl */
+ if(elem->len > 0) {
+ udata->iods[udata->idx].iod_size = (daos_size_t)(elem->len * udata->base_type_size);
+ daos_iov_set(&udata->sg_iovs[udata->idx], (void *)elem->p, udata->iods[udata->idx].iod_size);
+ } /* end if */
+ else {
+ udata->iods[udata->idx].iod_size = 0;
+ daos_iov_set(&udata->sg_iovs[udata->idx], NULL, 0);
+ } /* end else */
+ } /* end else */
+
+ /* Advance idx */
+ udata->idx++;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5VL_daosm_dataset_mem_vl_wr_cb() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5VL_daosm_dataset_write
*
* Purpose: Writes raw data from a buffer into a dataset.
@@ -4308,8 +4652,8 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5VL_daosm_dataset_write(void *_dset, hid_t H5_ATTR_UNUSED mem_type_id,
- hid_t mem_space_id, hid_t file_space_id, hid_t H5_ATTR_UNUSED dxpl_id,
+H5VL_daosm_dataset_write(void *_dset, hid_t mem_type_id, hid_t mem_space_id,
+ hid_t file_space_id, hid_t H5_ATTR_UNUSED dxpl_id,
const void *buf, void H5_ATTR_UNUSED **req)
{
H5VL_daosm_dset_t *dset = (H5VL_daosm_dset_t *)_dset;
@@ -4321,23 +4665,24 @@ H5VL_daosm_dataset_write(void *_dset, hid_t H5_ATTR_UNUSED mem_type_id,
hssize_t num_elem;
uint64_t chunk_coords[H5S_MAX_RANK];
daos_key_t dkey;
- daos_iod_t iod;
+ uint8_t **akeys = NULL;
+ daos_iod_t *iods = NULL;
+ daos_sg_list_t *sgls = NULL;
daos_recx_t recx;
daos_recx_t *recxs = &recx;
- daos_sg_list_t sgl;
daos_iov_t sg_iov;
daos_iov_t *sg_iovs = &sg_iov;
- size_t tot_nseq;
uint8_t dkey_buf[1 + H5S_MAX_RANK];
- uint8_t akey = H5VL_DAOSM_CHUNK_KEY;
- size_t file_type_size;
- size_t mem_type_size;
+ hid_t base_type_id = FAIL;
+ size_t base_type_size = 0;
void *tconv_buf = NULL;
void *bkg_buf = NULL;
- hbool_t fill_bkg = FALSE;
+ H5T_class_t type_class;
+ hbool_t is_vl = FALSE;
+ htri_t is_vl_str = FALSE;
uint8_t *p;
int ret;
- int i;
+ uint64_t i;
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
@@ -4366,116 +4711,206 @@ H5VL_daosm_dataset_write(void *_dset, hid_t H5_ATTR_UNUSED mem_type_id,
if((num_elem = H5Sget_select_npoints(real_mem_space_id)) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get number of points in selection")
- /* Initialize type conversion */
- if(H5VL_daosm_tconv_init(mem_type_id, &mem_type_size, dset->type_id, &file_type_size, (size_t)num_elem, &tconv_buf, &bkg_buf, NULL, &fill_bkg) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize type conversion")
-
/* Encode dkey (chunk coordinates). Prefix with '\0' to avoid accidental
* collisions with other d-keys in this object. For now just 1 chunk,
* starting at 0. */
HDmemset(chunk_coords, 0, sizeof(chunk_coords)); //DSMINC
p = dkey_buf;
*p++ = (uint8_t)'\0';
- for(i = 0; i < ndims; i++)
+ for(i = 0; i < (uint64_t)ndims; i++)
UINT64ENCODE(p, chunk_coords[i])
/* Set up dkey */
daos_iov_set(&dkey, dkey_buf, (daos_size_t)(1 + ((size_t)ndims * sizeof(chunk_coords[0]))));
- /* Set up iod */
- HDmemset(&iod, 0, sizeof(iod));
- daos_iov_set(&iod.iod_name, (void *)&akey, (daos_size_t)(sizeof(akey)));
- daos_csum_set(&iod.iod_kcsum, NULL, 0);
- iod.iod_size = file_type_size;
- iod.iod_type = DAOS_IOD_ARRAY;
+ /* Check for vlen */
+ if(H5T_NO_CLASS == (type_class = H5Tget_class(mem_type_id)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype class")
+ if(type_class == H5T_VLEN) {
+ is_vl = TRUE;
- /* Build recxs and sg_iovs */
- /* Get file dataspace object */
- if(NULL == (space = (H5S_t *)H5I_object(real_file_space_id)))
- HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+ /* Calculate base type size */
+ if((base_type_id = H5Tget_super(mem_type_id)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype base type")
+ if(0 == (base_type_size = H5Tget_size(base_type_id)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get datatype base type size")
+ } /* end if */
+ else if(type_class == H5T_STRING) {
+ /* check for vlen string */
+ if((is_vl_str = H5Tis_variable_str(mem_type_id)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't check for variable length string")
+ if(is_vl_str)
+ is_vl = TRUE;
+ } /* end if */
- /* Check for type conversion */
- if(tconv_buf) {
- /* Calculate recxs from file space */
- if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, (void *)buf, &recxs, NULL, &tot_nseq) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
- iod.iod_nr = (unsigned)tot_nseq;
- iod.iod_recxs = recxs;
+ /* Check for variable length */
+ if(is_vl) {
+ H5VL_daosm_vl_mem_ud_t mem_ud;
+ H5VL_daosm_vl_file_ud_t file_ud;
- /* Set up constant sgl info */
- sgl.sg_nr.num = 1;
- sgl.sg_iovs = &sg_iov;
+ /* Allocate array of akey pointers */
+ if(NULL == (akeys = (uint8_t **)H5MM_calloc((size_t)num_elem * sizeof(uint8_t *))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for akey array")
- /* Check if we need to fill background buffer */
- if(fill_bkg) {
- HDassert(bkg_buf);
+ /* Allocate array of iods */
+ if(NULL == (iods = (daos_iod_t *)H5MM_calloc((size_t)num_elem * sizeof(daos_iod_t))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for I/O descriptor array")
- /* Set sg_iov to point to background buffer */
- daos_iov_set(&sg_iov, bkg_buf, (daos_size_t)num_elem * (daos_size_t)file_type_size);
+ /* Allocate array of sg_iovs */
+ if(NULL == (sg_iovs = (daos_iov_t *)H5MM_malloc((size_t)num_elem * sizeof(daos_iov_t))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for scatter gather list")
- /* Read data from dataset to background buffer */
- if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*maps*/, NULL /*event*/)))
- HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data from dataset: %d", ret)
+ /* Allocate array of sgls */
+ if(NULL == (sgls = (daos_sg_list_t *)H5MM_malloc((size_t)num_elem * sizeof(daos_sg_list_t))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for scatter gather list array")
- /* Reset iod_size, if the dataset was not allocated then it could
- * have been overwritten by daos_obj_fetch */
- iod.iod_size = file_type_size;
- } /* end if */
+ /* Iterate over memory selection */
+ mem_ud.iods = iods;
+ mem_ud.sgls = sgls;
+ mem_ud.sg_iovs = sg_iovs;
+ mem_ud.is_vl_str = is_vl_str;
+ mem_ud.base_type_size = base_type_size;
+ mem_ud.idx = 0;
+ if(H5Diterate((void *)buf, mem_type_id, real_mem_space_id, H5VL_daosm_dataset_mem_vl_wr_cb, &mem_ud) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "memory selection iteration failed")
+ HDassert(mem_ud.idx == (uint64_t)num_elem);
+
+ /* Iterate over file selection. Note the bogus buffer and type_id,
+ * these don't matter since the "elem" parameter of the callback is not
+ * used. */
+ file_ud.akeys = akeys;
+ file_ud.iods = iods;
+ file_ud.idx = 0;
+ if(H5Diterate((void *)buf, mem_type_id, real_file_space_id, H5VL_daosm_dataset_file_vl_cb, &file_ud) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "file selection iteration failed")
+ HDassert(file_ud.idx == (uint64_t)num_elem);
+
+ /* Write data to dataset */
+ /* Note cast to unsigned reduces width to 32 bits. Should eventually
+ * check for overflow and iterate over 2^32 size blocks */
+ if(0 != (ret = daos_obj_update(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, (unsigned)num_elem, iods, sgls, NULL /*event*/)))
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data to dataset: %d", ret)
+ } /* end if */
+ else {
+ daos_iod_t iod;
+ daos_sg_list_t sgl;
+ uint8_t akey = H5VL_DAOSM_CHUNK_KEY;
+ size_t tot_nseq;
+ size_t file_type_size;
+ size_t mem_type_size;
+ hbool_t fill_bkg = FALSE;
- /* Gather data to conversion buffer */
- if(H5Dgather(real_mem_space_id, buf, mem_type_id, (size_t)num_elem * mem_type_size, tconv_buf, NULL, NULL) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't gather data to conversion buffer")
+ /* Initialize type conversion */
+ if(H5VL_daosm_tconv_init(mem_type_id, &mem_type_size, dset->type_id, &file_type_size, (size_t)num_elem, &tconv_buf, &bkg_buf, NULL, &fill_bkg) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize type conversion")
- /* Perform type conversion */
- if(H5Tconvert(mem_type_id, dset->type_id, (size_t)num_elem, tconv_buf, bkg_buf, dxpl_id) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't perform type conversion")
+ /* Set up iod */
+ HDmemset(&iod, 0, sizeof(iod));
+ daos_iov_set(&iod.iod_name, (void *)&akey, (daos_size_t)(sizeof(akey)));
+ daos_csum_set(&iod.iod_kcsum, NULL, 0);
+ iod.iod_size = file_type_size;
+ iod.iod_type = DAOS_IOD_ARRAY;
- /* Set sg_iovs to write from tconv_buf */
- daos_iov_set(&sg_iov, tconv_buf, (daos_size_t)num_elem * (daos_size_t)file_type_size);
- } /* end if */
- else {
- /* Check for memory space is H5S_ALL, use file space in this case */
- if(mem_space_id == H5S_ALL) {
- /* Calculate both recxs and sg_iovs at the same time from file space */
- if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, (void *)buf, &recxs, &sg_iovs, &tot_nseq) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
- iod.iod_nr = (unsigned)tot_nseq;
- sgl.sg_nr.num = (uint32_t)tot_nseq;
- } /* end if */
- else {
+ /* Build recxs and sg_iovs */
+ /* Get file dataspace object */
+ if(NULL == (space = (H5S_t *)H5I_object(real_file_space_id)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+
+ /* Check for type conversion */
+ if(tconv_buf) {
/* Calculate recxs from file space */
if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, (void *)buf, &recxs, NULL, &tot_nseq) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
iod.iod_nr = (unsigned)tot_nseq;
+ iod.iod_recxs = recxs;
- /* Get memory dataspace object */
- if(NULL == (space = (H5S_t *)H5I_object(real_mem_space_id)))
- HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+ /* Set up constant sgl info */
+ sgl.sg_nr.num = 1;
+ sgl.sg_iovs = &sg_iov;
- /* Calculate sg_iovs from mem space */
- if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, (void *)buf, NULL, &sg_iovs, &tot_nseq) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
- sgl.sg_nr.num = (uint32_t)tot_nseq;
+ /* Check if we need to fill background buffer */
+ if(fill_bkg) {
+ HDassert(bkg_buf);
+
+ /* Set sg_iov to point to background buffer */
+ daos_iov_set(&sg_iov, bkg_buf, (daos_size_t)num_elem * (daos_size_t)file_type_size);
+
+ /* Read data from dataset to background buffer */
+ if(0 != (ret = daos_obj_fetch(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*maps*/, NULL /*event*/)))
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data from dataset: %d", ret)
+
+ /* Reset iod_size, if the dataset was not allocated then it could
+ * have been overwritten by daos_obj_fetch */
+ iod.iod_size = file_type_size;
+ } /* end if */
+
+ /* Gather data to conversion buffer */
+ if(H5Dgather(real_mem_space_id, buf, mem_type_id, (size_t)num_elem * mem_type_size, tconv_buf, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't gather data to conversion buffer")
+
+ /* Perform type conversion */
+ if(H5Tconvert(mem_type_id, dset->type_id, (size_t)num_elem, tconv_buf, bkg_buf, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't perform type conversion")
+
+ /* Set sg_iovs to write from tconv_buf */
+ daos_iov_set(&sg_iov, tconv_buf, (daos_size_t)num_elem * (daos_size_t)file_type_size);
+ } /* end if */
+ else {
+ /* Check for memory space is H5S_ALL, use file space in this case */
+ if(mem_space_id == H5S_ALL) {
+ /* Calculate both recxs and sg_iovs at the same time from file space */
+ if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, (void *)buf, &recxs, &sg_iovs, &tot_nseq) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
+ iod.iod_nr = (unsigned)tot_nseq;
+ sgl.sg_nr.num = (uint32_t)tot_nseq;
+ } /* end if */
+ else {
+ /* Calculate recxs from file space */
+ if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, (void *)buf, &recxs, NULL, &tot_nseq) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
+ iod.iod_nr = (unsigned)tot_nseq;
+
+ /* Get memory dataspace object */
+ if(NULL == (space = (H5S_t *)H5I_object(real_mem_space_id)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+
+ /* Calculate sg_iovs from mem space */
+ if(H5VL_daosm_sel_to_recx_iov(space, file_type_size, (void *)buf, NULL, &sg_iovs, &tot_nseq) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't generate sequence lists for DAOS I/O")
+ sgl.sg_nr.num = (uint32_t)tot_nseq;
+ } /* end else */
+
+ /* Point iod and sgl to lists generated above */
+ iod.iod_recxs = recxs;
+ sgl.sg_iovs = sg_iovs;
} /* end else */
- /* Point iod and sgl to lists generated above */
- iod.iod_recxs = recxs;
- sgl.sg_iovs = sg_iovs;
+ /* Write data to dataset */
+ if(0 != (ret = daos_obj_update(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*event*/)))
+ HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data to dataset: %d", ret)
} /* end else */
- /* Write data to dataset */
- if(0 != (ret = daos_obj_update(dset->obj.obj_oh, dset->obj.item.file->epoch, &dkey, 1, &iod, &sgl, NULL /*event*/)))
- HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data to dataset: %d", ret)
-
done:
/* Free memory */
+ iods = (daos_iod_t *)H5MM_xfree(iods);
if(recxs != &recx)
H5MM_free(recxs);
- if(sg_iovs != &sg_iov)
+ sgls = (daos_sg_list_t *)H5MM_xfree(sgls);
+ if(sg_iovs && (sg_iovs != &sg_iov))
H5MM_free(sg_iovs);
tconv_buf = H5MM_xfree(tconv_buf);
bkg_buf = H5MM_xfree(bkg_buf);
+ if(akeys) {
+ for(i = 0; i < (uint64_t)num_elem; i++)
+ H5MM_xfree(akeys[i]);
+ H5MM_free(akeys);
+ } /* end if */
+
+ if(base_type_id != FAIL)
+ if(H5I_dec_app_ref(base_type_id) < 0)
+ HDONE_ERROR(H5E_ATTR, H5E_CLOSEERROR, FAIL, "can't close base type id")
+
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5VL_daosm_dataset_write() */
@@ -5330,7 +5765,6 @@ H5VL_daosm_attribute_read(void *_attr, hid_t mem_type_id, void *buf,
daos_iov_t *sg_iovs = NULL;
char attr_key[] = H5VL_DAOSM_ATTR_KEY;
hid_t base_type_id = FAIL;
- size_t file_type_size;
size_t base_type_size = 0;
uint64_t attr_size;
void *tconv_buf = NULL;
@@ -5393,7 +5827,7 @@ H5VL_daosm_attribute_read(void *_attr, hid_t mem_type_id, void *buf,
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for akey array")
/* Allocate array of iods */
- if(NULL == (iods = (daos_iod_t *)H5MM_malloc(attr_size * sizeof(daos_iod_t))))
+ if(NULL == (iods = (daos_iod_t *)H5MM_calloc(attr_size * sizeof(daos_iod_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for I/O descriptor array")
/* First loop over elements, set up operation to read vl sizes */
@@ -5408,7 +5842,6 @@ H5VL_daosm_attribute_read(void *_attr, hid_t mem_type_id, void *buf,
UINT64ENCODE(p, i)
/* Set up iod. Use "single" records of varying size. */
- HDmemset(&iods[i], 0, sizeof(iods[0]));
daos_iov_set(&iods[i].iod_name, (void *)akeys[i], (daos_size_t)akey_len);
daos_csum_set(&iods[i].iod_kcsum, NULL, 0);
iods[i].iod_nr = 1u;
@@ -5498,6 +5931,7 @@ H5VL_daosm_attribute_read(void *_attr, hid_t mem_type_id, void *buf,
daos_sg_list_t sgl;
daos_iov_t sg_iov;
size_t mem_type_size;
+ size_t file_type_size;
H5VL_daosm_tconv_reuse_t reuse = H5VL_DAOSM_TCONV_REUSE_NONE;
hbool_t fill_bkg = FALSE;
@@ -5597,8 +6031,8 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5VL_daosm_attribute_write(void *_attr, hid_t H5_ATTR_UNUSED mem_type_id,
- const void *buf, hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR_UNUSED **req)
+H5VL_daosm_attribute_write(void *_attr, hid_t mem_type_id, const void *buf,
+ hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR_UNUSED **req)
{
H5VL_daosm_attr_t *attr = (H5VL_daosm_attr_t *)_attr;
int ndims;
@@ -5612,7 +6046,6 @@ H5VL_daosm_attribute_write(void *_attr, hid_t H5_ATTR_UNUSED mem_type_id,
daos_iov_t *sg_iovs = NULL;
char attr_key[] = H5VL_DAOSM_ATTR_KEY;
hid_t base_type_id = FAIL;
- size_t file_type_size;
size_t base_type_size = 0;
uint64_t attr_size;
void *tconv_buf = NULL;
@@ -5678,7 +6111,7 @@ H5VL_daosm_attribute_write(void *_attr, hid_t H5_ATTR_UNUSED mem_type_id,
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for akey array")
/* Allocate array of iods */
- if(NULL == (iods = (daos_iod_t *)H5MM_malloc(attr_size * sizeof(daos_iod_t))))
+ if(NULL == (iods = (daos_iod_t *)H5MM_calloc(attr_size * sizeof(daos_iod_t))))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for I/O descriptor array")
/* Allocate array of sg_iovs */
@@ -5702,7 +6135,6 @@ H5VL_daosm_attribute_write(void *_attr, hid_t H5_ATTR_UNUSED mem_type_id,
/* Set up iod, determine size below. Use "single" records of
* varying size. */
- HDmemset(&iods[i], 0, sizeof(iods[0]));
daos_iov_set(&iods[i].iod_name, (void *)akeys[i], (daos_size_t)akey_len);
daos_csum_set(&iods[i].iod_kcsum, NULL, 0);
iods[i].iod_nr = 1u;
@@ -5763,6 +6195,7 @@ H5VL_daosm_attribute_write(void *_attr, hid_t H5_ATTR_UNUSED mem_type_id,
daos_sg_list_t sgl;
daos_iov_t sg_iov;
size_t mem_type_size;
+ size_t file_type_size;
hbool_t fill_bkg = FALSE;
/* Check for type conversion */