From 50a7ed337ce5bb10f9ad86ef02d580599e2101bf Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Tue, 12 Feb 2019 22:21:11 -0800 Subject: * Added a script to run the RADOS examples in order. * Updated BRANCH.txt with useful information about building and testing. * Makefile.am now installs everything you need as well as the test script. --- BRANCH.txt | 55 ++++++++++++++++++++++++++------ examples/Makefile.am | 3 +- examples/run_rados_examples.sh | 71 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 11 deletions(-) create mode 100755 examples/run_rados_examples.sh diff --git a/BRANCH.txt b/BRANCH.txt index 2cbc975..57aa2da 100644 --- a/BRANCH.txt +++ b/BRANCH.txt @@ -1,15 +1,50 @@ -autotools_rework +This branch is a copy of Neil's RADOS VOL connector branch. +Very few changes have been made to the branch so it stays +as a 'known good' version of the connector. -The purpose of this branch is to make changes to the autotools files, -particularly configure.ac. This will remove a lot of old cruft and update -the input files. This effort was started in January 2015 and should be -completed within the year. +This is based off the ancient 'vol' branch and is wildy +out of sync with develop. You will not be able to move +the RADOS VOL connector built here to HDF5 1.12.0 without +a lot of surgery. The rados_vol branch in this repository +is kept in sync with develop and is probably the one +you want for that. -This branch tracks the trunk. +Note that this is a dead branch, so no effort has been +made to keep the MANIFEST sane, etc. -Dana Robinson and Allen Byrne are the owners of this branch. +************ +* BUILDING * +************ -This branch should be closed when the bulk of the autotools work is -complete and has been merged into the trunk and 1.8 branch. This -work should be completed by mid-2015. +The code for the RADOS VOL connector lies entirely in +the H5VLrados* files in the src directory. The connector +is built as a static part of the library. +I build this branch on Ubuntu, with the system default +librados-dev package and its dependencies installed. I use +the autotools and the only thing I have to specify is the +linker flags: + +LDFLAGS="-L/path/to/librados -lrados" + +YMMV, but the autotools do no librados hunting so you may +need to specify things it can't figure out on its own. + + +*********** +* TESTING * +*********** + +There are no tests for the RADOS VOL connector in the main +library, so running 'make check' won't provide much +information. + +Once the library has been built, you can run some smoke +checks by running 'make install' and going to the +share/hdf5_examples/c/ directory and running the +run_rados_examples.sh script found there. That will +create a small pool (or clear it out if it exists) +and run all the example scripts in a sane order. + +*** YOU WILL NEED TO COPY YOUR ceph.conf FILE +*** INTO THIS DIRECTORY FIRST!!! diff --git a/examples/Makefile.am b/examples/Makefile.am index 323541f..ae0ac65 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -63,7 +63,8 @@ INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c \ h5rados_file_create.c h5rados_file_open.c h5rados_group_create.c \ h5rados_group_open.c h5rados_dset_create.c \ h5rados_dset_open.c h5rados_dset_write.c h5rados_dset_read.c \ - h5rados_dset_wpartial.c h5rados_dset_rpartial.c + h5rados_dset_wpartial.c h5rados_dset_rpartial.c \ + h5rados_example.h run_rados_examples.sh diff --git a/examples/run_rados_examples.sh b/examples/run_rados_examples.sh new file mode 100755 index 0000000..7782a56 --- /dev/null +++ b/examples/run_rados_examples.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Build RADOS examples. +# Assumes RADOS can be found in the usual location. +echo "Building RADOS examples" +../../../bin/h5pcc -o h5rados_file_create h5rados_file_create.c -lrados +../../../bin/h5pcc -o h5rados_file_open h5rados_file_open.c -lrados +../../../bin/h5pcc -o h5rados_dset_create h5rados_dset_create.c -lrados +../../../bin/h5pcc -o h5rados_dset_open h5rados_dset_open.c -lrados +../../../bin/h5pcc -o h5rados_group_create h5rados_group_create.c -lrados +../../../bin/h5pcc -o h5rados_group_open h5rados_group_open.c -lrados +../../../bin/h5pcc -o h5rados_dset_write h5rados_dset_write.c -lrados +../../../bin/h5pcc -o h5rados_dset_read h5rados_dset_read.c -lrados + +../../../bin/h5pcc -o h5rados_dset_rpartial h5rados_dset_rpartial.c -lrados +../../../bin/h5pcc -o h5rados_dset_wpartial h5rados_dset_wpartial.c -lrados +echo "DONE" +echo + +# Create the pool +echo "Creating the test pool" +echo "(Ceph makes it hard to delete pools, so we never do that" +echo "and you may see a message that the pool already exists.)" +ceph osd pool create mypool 128 +echo + +# Dump some Ceph info +echo "Pool status at start of tests:" +rados -p mypool ls +echo + +# Clean out the test pool +echo "Cleaning out the test pool" +for i in $(rados -p mypool ls); do echo $i; rados -p mypool rm $i; done +echo + +# Run the RADOS examples +echo "Running RADOS examples" +# File create/open +echo "./h5rados_file_create testfile" +mpiexec -n 2 ./h5rados_file_create testfile +echo "./h5rados_file_open testfile" +mpiexec -n 2 ./h5rados_file_open testfile +# Dataset create/open +echo "./h5rados_dset_create testfile testdset" +mpiexec -n 2 ./h5rados_dset_create testfile testdset +echo "./h5rados_dset_open testfile testdset" +mpiexec -n 2 ./h5rados_dset_open testfile testdset +# Group create/open +echo "./h5rados_group_create testfile testgroup" +mpiexec -n 2 ./h5rados_group_create testfile testgroup +echo "./h5rados_group_open testfile testgroup" +mpiexec -n 2 ./h5rados_group_open testfile testgroup +# Dataset read/write +echo "./h5rados_dset_write testfile testdset" +mpiexec -n 2 ./h5rados_dset_write testfile testdset +echo "./h5rados_dset_read testfile testdset" +mpiexec -n 2 ./h5rados_dset_read testfile testdset +# Dataset read/write (partial) +echo "./h5rados_dset_wpartial testfile testdset" +mpiexec -n 2 ./h5rados_dset_wpartial testfile testdset +echo "./h5rados_dset_rpartial testfile testdset" +mpiexec -n 2 ./h5rados_dset_rpartial testfile testdset +echo + + +# Dump the Ceph info again +echo "Pool status at end of tests:" +rados -p mypool ls +echo + -- cgit v0.12 From a3cc965997df8fedad5dab12cf884c076ddde238 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 18 Feb 2019 16:24:42 -0800 Subject: Converted one of the stats calls to use a read op instead. --- src/H5VLrados.c | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/H5VLrados.c b/src/H5VLrados.c index 76a865e..9e7fbce 100644 --- a/src/H5VLrados.c +++ b/src/H5VLrados.c @@ -1946,8 +1946,8 @@ H5VL_rados_group_open_helper(H5VL_rados_file_t *file, uint64_t oid, { H5VL_rados_group_t *grp = NULL; void *gcpl_buf = NULL; - uint64_t gcpl_len; - time_t pmtime; + uint64_t gcpl_len = 0; + time_t pmtime = 0; int ret; void *ret_value = NULL; @@ -1965,9 +1965,37 @@ H5VL_rados_group_open_helper(H5VL_rados_file_t *file, uint64_t oid, grp->gcpl_id = FAIL; grp->gapl_id = FAIL; +#if 1 /* fancy new read-op-based code */ +{ + rados_read_op_t read_op; + hbool_t read_op_init = FALSE; + int prval; + + /* Create read op */ + if(NULL == (read_op = rados_create_read_op())) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't create read operation") + read_op_init = TRUE; + + /* Add the get stats operation (returns void) */ + rados_read_op_stat(read_op, &gcpl_len, &pmtime, &prval); + + /* Execute read operation */ + if((ret = rados_read_op_operate(read_op, ioctx_g, grp->obj.oid, LIBRADOS_OPERATION_NOFLAG)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't perform read operation: %s", strerror(-ret)) + if(0 < prval) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, NULL, "stats not found for object") + + /* clean up */ + if(read_op_init) + rados_release_read_op(read_op); +} +#endif + +#if 0 /* old code */ /* Read internal metadata size from group */ if((ret = rados_stat(ioctx_g, grp->obj.oid, &gcpl_len, &pmtime)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, NULL, "can't read metadata size from group: %s", strerror(-ret)) +#endif /* Check for metadata not found */ if(gcpl_len == (uint64_t)0) -- cgit v0.12 From efb2f4932fc622e68b1d6e27435beb06da617638 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 18 Feb 2019 16:45:36 -0800 Subject: Both rados_stats calls are wrapped. You can #define OLD_RADOS_CALLS to enable the old way of doing things. --- src/H5VLrados.c | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/H5VLrados.c b/src/H5VLrados.c index 9e7fbce..bfeb37d 100644 --- a/src/H5VLrados.c +++ b/src/H5VLrados.c @@ -1965,7 +1965,11 @@ H5VL_rados_group_open_helper(H5VL_rados_file_t *file, uint64_t oid, grp->gcpl_id = FAIL; grp->gapl_id = FAIL; -#if 1 /* fancy new read-op-based code */ +#ifdef OLD_RADOS_CALLS + /* Read internal metadata size from group */ + if((ret = rados_stat(ioctx_g, grp->obj.oid, &gcpl_len, &pmtime)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, NULL, "can't read metadata size from group: %s", strerror(-ret)) +#else { rados_read_op_t read_op; hbool_t read_op_init = FALSE; @@ -1991,12 +1995,6 @@ H5VL_rados_group_open_helper(H5VL_rados_file_t *file, uint64_t oid, } #endif -#if 0 /* old code */ - /* Read internal metadata size from group */ - if((ret = rados_stat(ioctx_g, grp->obj.oid, &gcpl_len, &pmtime)) < 0) - HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, NULL, "can't read metadata size from group: %s", strerror(-ret)) -#endif - /* Check for metadata not found */ if(gcpl_len == (uint64_t)0) HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, NULL, "internal metadata not found") @@ -2774,7 +2772,7 @@ H5VL_rados_dataset_open(void *_item, uint64_t type_len = 0; uint64_t space_len = 0; uint64_t dcpl_len = 0; - time_t pmtime; + time_t pmtime = 0; uint8_t dinfo_buf_static[H5VL_RADOS_DINFO_BUF_SIZE]; uint8_t *dinfo_buf_dyn = NULL; uint8_t *dinfo_buf = dinfo_buf_static; @@ -2832,9 +2830,36 @@ H5VL_rados_dataset_open(void *_item, HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "can't encode string oid") } /* end else */ +#ifdef OLD_RADOS_CALLS /* Read internal metadata size from dataset */ if((ret = rados_stat(ioctx_g, dset->obj.oid, &md_len, &pmtime)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, NULL, "can't read metadata size from group: %s", strerror(-ret)) +#else +{ + rados_read_op_t read_op; + hbool_t read_op_init = FALSE; + int prval; + + /* Create read op */ + if(NULL == (read_op = rados_create_read_op())) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't create read operation") + read_op_init = TRUE; + + /* Add the get stats operation (returns void) */ + rados_read_op_stat(read_op, &md_len, &pmtime, &prval); + + /* Execute read operation */ + if((ret = rados_read_op_operate(read_op, ioctx_g, dset->obj.oid, LIBRADOS_OPERATION_NOFLAG)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't perform read operation: %s", strerror(-ret)) + if(0 < prval) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, NULL, "stats not found for object") + + /* clean up */ + if(read_op_init) + rados_release_read_op(read_op); +} +#endif + /* Check for metadata not found */ if(md_len == (uint64_t)0) -- cgit v0.12 From fedf16540c21c722cb1e6b2017ed5ef1417face7 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 18 Feb 2019 20:17:54 -0800 Subject: Added a helper function to wrap rados_write_full(). --- src/H5VLrados.c | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/H5VLrados.c b/src/H5VLrados.c index bfeb37d..4069d0b 100644 --- a/src/H5VLrados.c +++ b/src/H5VLrados.c @@ -348,6 +348,36 @@ rados_ioctx_t ioctx_g; hbool_t ioctx_init_g = FALSE; +/*------------------------------------------------------------------------- + * Function: H5VL_rados_write_full + * + * Purpose: Recreates rados_write_full(). We are trying to use all + * read_op calls and this lets us do that without duplicating + * all the RADOS calls. + * + * Return: Success: 0 + * Failure: -11 + * + * Programmer: Dana Robinson + * February, 2019 + * + *------------------------------------------------------------------------- + */ +static int +H5VL_rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len) +{ + int ret; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + if((ret = rados_write_full(io, oid, buf, len)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't write metadata to object: %s", strerror(-ret)) +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5VL_rados_write_full() */ + + /* Create a RADOS string oid given the file name and binary oid */ static herr_t H5VL_rados_oid_create_string(const H5VL_rados_file_t *file, uint64_t bin_oid, @@ -1272,7 +1302,7 @@ H5VL_rados_write_max_oid(H5VL_rados_file_t *file) UINT64ENCODE(p, file->max_oid) - if((ret = rados_write_full(ioctx_g, file->glob_md_oid, (const char *)wbuf, (size_t)8)) < 0) + if((ret = H5VL_rados_write_full(ioctx_g, file->glob_md_oid, (const char *)wbuf, (size_t)8)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't write metadata to group: %s", strerror(-ret)) file->max_oid_dirty = FALSE; } /* end if */ @@ -1822,7 +1852,7 @@ H5VL_rados_group_create_helper(H5VL_rados_file_t *file, hid_t gcpl_id, HGOTO_ERROR(H5E_SYM, H5E_CANTENCODE, NULL, "can't serialize gcpl") /* Write internal metadata to group */ - if((ret = rados_write_full(ioctx_g, grp->obj.oid, gcpl_buf, gcpl_size)) < 0) + if((ret = H5VL_rados_write_full(ioctx_g, grp->obj.oid, gcpl_buf, gcpl_size)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't write metadata to group: %s", strerror(-ret)) /* Mark max OID as dirty */ @@ -2700,7 +2730,7 @@ H5VL_rados_dataset_create(void *_item, HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, NULL, "can't serialize dcpl") /* Write internal metadata to dataset */ - if((ret = rados_write_full(ioctx_g, dset->obj.oid, (const char *)md_buf, md_size)) < 0) + if((ret = H5VL_rados_write_full(ioctx_g, dset->obj.oid, (const char *)md_buf, md_size)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "can't write metadata to dataset: %s", strerror(-ret)) /* Mark max OID as dirty */ -- cgit v0.12 From 463703366250bfbcf4f9aa0d27f41b8d8f7a1afe Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 18 Feb 2019 20:34:08 -0800 Subject: rados_write_full calls now use write_ops. --- src/H5VLrados.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/H5VLrados.c b/src/H5VLrados.c index 4069d0b..bb0ef72 100644 --- a/src/H5VLrados.c +++ b/src/H5VLrados.c @@ -366,13 +366,36 @@ hbool_t ioctx_init_g = FALSE; static int H5VL_rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len) { - int ret; + int ret = 0; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT +#ifdef OLD_RADOS_CALLS if((ret = rados_write_full(io, oid, buf, len)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't write metadata to object: %s", strerror(-ret)) +#else +{ + rados_write_op_t write_op; + hbool_t write_op_init = FALSE; + + /* Create write op */ + if(NULL == (write_op = rados_create_write_op())) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't create write operation") + write_op_init = TRUE; + + /* Add the write full operation (returns void) */ + rados_write_op_write_full(write_op, buf, len); + + /* Execute write operation */ + if((ret = rados_write_op_operate(write_op, io, oid, NULL, LIBRADOS_OPERATION_NOFLAG)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't perform write operation: %s", strerror(-ret)) + + /* clean up */ + if(write_op_init) + rados_release_write_op(write_op); +} +#endif done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_rados_write_full() */ -- cgit v0.12 From 9c6989cb7d953c62ed3d0813ec76a7f712365ba3 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 18 Feb 2019 21:50:15 -0800 Subject: * Added a rados_read() wrapper function (still uses rados_read()). * Some cleanup in the write wrapper. --- src/H5VLrados.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/src/H5VLrados.c b/src/H5VLrados.c index bb0ef72..a0a4111 100644 --- a/src/H5VLrados.c +++ b/src/H5VLrados.c @@ -200,6 +200,12 @@ static herr_t H5VL_rados_file_flush(H5VL_rados_file_t *file); static herr_t H5VL_rados_file_close_helper(H5VL_rados_file_t *file, hid_t dxpl_id, void **req); +/* read/write_op equivalents for some RADOS calls */ +static int H5VL_rados_read(rados_ioctx_t io, const char *oid, char *buf, + size_t len, uint64_t off); +static int H5VL_rados_write_full(rados_ioctx_t io, const char *oid, + const char *buf, size_t len); + static herr_t H5VL_rados_link_read(H5VL_rados_group_t *grp, const char *name, H5VL_rados_link_val_t *val); static herr_t H5VL_rados_link_write(H5VL_rados_group_t *grp, const char *name, @@ -349,14 +355,45 @@ hbool_t ioctx_init_g = FALSE; /*------------------------------------------------------------------------- + * Function: H5VL_rados_read + * + * Purpose: Recreates rados_read(). We are trying to use all read_op + * calls and this lets us do that without duplicating all + * the RADOS boilerplate. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * February, 2019 + * + *------------------------------------------------------------------------- + */ +static int +H5VL_rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64_t off) +{ + int ret = 0; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT + + /* Read data from the object */ + if((ret = rados_read(io, oid, buf, len, off)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, (-1), "can't read metadata from dataset: %s", strerror(-ret)) +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5VL_rados_read() */ + + +/*------------------------------------------------------------------------- * Function: H5VL_rados_write_full * * Purpose: Recreates rados_write_full(). We are trying to use all - * read_op calls and this lets us do that without duplicating - * all the RADOS calls. + * write_op calls and this lets us do that without duplicating + * all the RADOS boilerplate. * * Return: Success: 0 - * Failure: -11 + * Failure: -1 * * Programmer: Dana Robinson * February, 2019 @@ -367,13 +404,13 @@ static int H5VL_rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len) { int ret = 0; - herr_t ret_value = SUCCEED; + int ret_value = 0; FUNC_ENTER_NOAPI_NOINIT #ifdef OLD_RADOS_CALLS if((ret = rados_write_full(io, oid, buf, len)) < 0) - HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't write metadata to object: %s", strerror(-ret)) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, (-1), "can't write metadata to object: %s", strerror(-ret)) #else { rados_write_op_t write_op; @@ -381,7 +418,7 @@ H5VL_rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t /* Create write op */ if(NULL == (write_op = rados_create_write_op())) - HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't create write operation") + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, (-1), "can't create write operation") write_op_init = TRUE; /* Add the write full operation (returns void) */ @@ -389,7 +426,7 @@ H5VL_rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t /* Execute write operation */ if((ret = rados_write_op_operate(write_op, io, oid, NULL, LIBRADOS_OPERATION_NOFLAG)) < 0) - HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't perform write operation: %s", strerror(-ret)) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, (-1), "can't perform write operation: %s", strerror(-ret)) /* clean up */ if(write_op_init) @@ -1017,7 +1054,7 @@ H5VL_rados_file_open(const char *name, unsigned flags, hid_t fapl_id, /* Read max oid directly to foi_buf */ /* Check for does not exist here and assume 0? -NAF */ - if((ret = rados_read(ioctx_g, file->glob_md_oid, foi_buf, 8, 0)) < 0) + if((ret = H5VL_rados_read(ioctx_g, file->glob_md_oid, foi_buf, 8, 0)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, NULL, "can't read metadata from dataset: %s", strerror(-ret)) /* Decode max oid */ @@ -2057,7 +2094,7 @@ H5VL_rados_group_open_helper(H5VL_rados_file_t *file, uint64_t oid, HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't allocate buffer for serialized gcpl") /* Read internal metadata from group */ - if((ret = rados_read(ioctx_g, grp->obj.oid, gcpl_buf, gcpl_len, 0)) < 0) + if((ret = H5VL_rados_read(ioctx_g, grp->obj.oid, gcpl_buf, gcpl_len, 0)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, NULL, "can't read metadata from group: %s", strerror(-ret)) /* Decode GCPL */ @@ -2926,7 +2963,7 @@ H5VL_rados_dataset_open(void *_item, } /* end if */ /* Read internal metadata from dataset */ - if((ret = rados_read(ioctx_g, dset->obj.oid, (char *)(dinfo_buf + sizeof(uint64_t)), md_len, 0)) < 0) + if((ret = H5VL_rados_read(ioctx_g, dset->obj.oid, (char *)(dinfo_buf + sizeof(uint64_t)), md_len, 0)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, NULL, "can't read metadata from dataset: %s", strerror(-ret)) /* Decode info lengths */ -- cgit v0.12 From 0ed62231af2d9d518b94deefd7573e0c01f8ebfa Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Mon, 18 Feb 2019 22:02:01 -0800 Subject: Finished implementation of RADOS wrappers for calls not implemented in mobject. --- src/H5VLrados.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/H5VLrados.c b/src/H5VLrados.c index a0a4111..c445ab9 100644 --- a/src/H5VLrados.c +++ b/src/H5VLrados.c @@ -377,9 +377,39 @@ H5VL_rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64 FUNC_ENTER_NOAPI_NOINIT +#ifdef OLD_RADOS_CALLS /* Read data from the object */ if((ret = rados_read(io, oid, buf, len, off)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, (-1), "can't read metadata from dataset: %s", strerror(-ret)) +#else +{ + rados_read_op_t read_op; + hbool_t read_op_init = FALSE; + size_t bytes_read = 0; + int prval; + + /* Create read op */ + if(NULL == (read_op = rados_create_read_op())) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, (-1), "can't create read operation") + read_op_init = TRUE; + + /* Add the read operation (returns void) */ + rados_read_op_read(read_op, off, len, buf, &bytes_read, &prval); + + /* Execute read operation */ + if((ret = rados_read_op_operate(read_op, io, oid, LIBRADOS_OPERATION_NOFLAG)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, (-1), "can't perform read operation: %s", strerror(-ret)) + if(0 < prval) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, (-1), "RADOS read operation failed for object") + if(0 == bytes_read) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, (-1), "metadata not found for object") + + /* clean up */ + if(read_op_init) + rados_release_read_op(read_op); +} +#endif + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_rados_read() */ @@ -433,6 +463,7 @@ H5VL_rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t rados_release_write_op(write_op); } #endif + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_rados_write_full() */ -- cgit v0.12 From c2bc773474e9215eea72dc2bf90a7208467fe540 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Tue, 19 Feb 2019 14:44:12 -0800 Subject: Moved the rados_stat() modifications into a helper function. --- src/H5VLrados.c | 126 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/src/H5VLrados.c b/src/H5VLrados.c index c445ab9..0ee5c92 100644 --- a/src/H5VLrados.c +++ b/src/H5VLrados.c @@ -201,10 +201,9 @@ static herr_t H5VL_rados_file_close_helper(H5VL_rados_file_t *file, hid_t dxpl_id, void **req); /* read/write_op equivalents for some RADOS calls */ -static int H5VL_rados_read(rados_ioctx_t io, const char *oid, char *buf, - size_t len, uint64_t off); -static int H5VL_rados_write_full(rados_ioctx_t io, const char *oid, - const char *buf, size_t len); +static int H5VL_rados_read(rados_ioctx_t io, const char *oid, char *buf, size_t len, uint64_t off); +static int H5VL_rados_write_full(rados_ioctx_t io, const char *oid, const char *buf, size_t len); +static int H5VL_rados_stat(rados_ioctx_t io, const char *oid, uint64_t *psize, time_t * pmtime); static herr_t H5VL_rados_link_read(H5VL_rados_group_t *grp, const char *name, H5VL_rados_link_val_t *val); @@ -469,6 +468,64 @@ done: } /* end H5VL_rados_write_full() */ +/*------------------------------------------------------------------------- + * Function: H5VL_rados_stat + * + * Purpose: Recreates rados_stat(). We are trying to use all read_op + * calls and this lets us do that without duplicating all + * the RADOS boilerplate. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * February, 2019 + * + *------------------------------------------------------------------------- + */ +static int +H5VL_rados_stat(rados_ioctx_t io, const char *oid, uint64_t *psize, time_t * pmtime) +{ + int ret = 0; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT + +#ifdef OLD_RADOS_CALLS + /* Get the object size and time */ + if((ret = rados_stat(io, oid, psize, pmtime)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, (-1), "can't read object size and time: %s", strerror(-ret)) +#else +{ + rados_read_op_t read_op; + hbool_t read_op_init = FALSE; + int prval; + + /* Create read op */ + if(NULL == (read_op = rados_create_read_op())) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, (-1), "can't create read operation") + read_op_init = TRUE; + + /* Add the get stats operation (returns void) */ + rados_read_op_stat(read_op, psize, pmtime, &prval); + + /* Execute read operation */ + if((ret = rados_read_op_operate(read_op, io, oid, LIBRADOS_OPERATION_NOFLAG)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, (-1), "can't perform read operation: %s", strerror(-ret)) + if(0 < prval) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, (-1), "stats not found for object") + + /* clean up */ + if(read_op_init) + rados_release_read_op(read_op); +} +#endif + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5VL_rados_stat() */ + + /* Create a RADOS string oid given the file name and binary oid */ static herr_t H5VL_rados_oid_create_string(const H5VL_rados_file_t *file, uint64_t bin_oid, @@ -2086,35 +2143,9 @@ H5VL_rados_group_open_helper(H5VL_rados_file_t *file, uint64_t oid, grp->gcpl_id = FAIL; grp->gapl_id = FAIL; -#ifdef OLD_RADOS_CALLS - /* Read internal metadata size from group */ - if((ret = rados_stat(ioctx_g, grp->obj.oid, &gcpl_len, &pmtime)) < 0) + /* Get the object size and time */ + if((ret = H5VL_rados_stat(ioctx_g, grp->obj.oid, &gcpl_len, &pmtime)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, NULL, "can't read metadata size from group: %s", strerror(-ret)) -#else -{ - rados_read_op_t read_op; - hbool_t read_op_init = FALSE; - int prval; - - /* Create read op */ - if(NULL == (read_op = rados_create_read_op())) - HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't create read operation") - read_op_init = TRUE; - - /* Add the get stats operation (returns void) */ - rados_read_op_stat(read_op, &gcpl_len, &pmtime, &prval); - - /* Execute read operation */ - if((ret = rados_read_op_operate(read_op, ioctx_g, grp->obj.oid, LIBRADOS_OPERATION_NOFLAG)) < 0) - HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't perform read operation: %s", strerror(-ret)) - if(0 < prval) - HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, NULL, "stats not found for object") - - /* clean up */ - if(read_op_init) - rados_release_read_op(read_op); -} -#endif /* Check for metadata not found */ if(gcpl_len == (uint64_t)0) @@ -2951,36 +2982,9 @@ H5VL_rados_dataset_open(void *_item, HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "can't encode string oid") } /* end else */ -#ifdef OLD_RADOS_CALLS - /* Read internal metadata size from dataset */ - if((ret = rados_stat(ioctx_g, dset->obj.oid, &md_len, &pmtime)) < 0) + /* Get the object size and time */ + if((ret = H5VL_rados_stat(ioctx_g, dset->obj.oid, &md_len, &pmtime)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, NULL, "can't read metadata size from group: %s", strerror(-ret)) -#else -{ - rados_read_op_t read_op; - hbool_t read_op_init = FALSE; - int prval; - - /* Create read op */ - if(NULL == (read_op = rados_create_read_op())) - HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't create read operation") - read_op_init = TRUE; - - /* Add the get stats operation (returns void) */ - rados_read_op_stat(read_op, &md_len, &pmtime, &prval); - - /* Execute read operation */ - if((ret = rados_read_op_operate(read_op, ioctx_g, dset->obj.oid, LIBRADOS_OPERATION_NOFLAG)) < 0) - HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "can't perform read operation: %s", strerror(-ret)) - if(0 < prval) - HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, NULL, "stats not found for object") - - /* clean up */ - if(read_op_init) - rados_release_read_op(read_op); -} -#endif - /* Check for metadata not found */ if(md_len == (uint64_t)0) -- cgit v0.12 From 9e5651ec4f0073ed84a5459dc109ca0d6b602862 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Wed, 20 Feb 2019 12:43:22 -0800 Subject: Added a RADOS/mobject #ifdef switch. --- src/H5VLrados_public.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/H5VLrados_public.h b/src/H5VLrados_public.h index 65ee9f4..e49339b 100644 --- a/src/H5VLrados_public.h +++ b/src/H5VLrados_public.h @@ -25,7 +25,11 @@ #define H5_HAVE_EFF 1 /* DSMINC */ /* External headers needed by this file */ +#ifdef HDF5_USE_MOBJECT +#include +#else #include +#endif /* Public headers needed by this file */ #include "H5public.h" -- cgit v0.12 From 0e402e436d0c58acc72e45ab6153d5662b3148e0 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Fri, 22 Feb 2019 12:53:07 -0600 Subject: Updated the tests to use mobject. --- BRANCH.txt | 37 +++++++++++++++++++++++++++++++ examples/Makefile.am | 2 +- examples/h5rados_dset_create.c | 2 +- examples/h5rados_dset_open.c | 2 +- examples/h5rados_dset_read.c | 2 +- examples/h5rados_dset_rpartial.c | 2 +- examples/h5rados_dset_wpartial.c | 2 +- examples/h5rados_dset_write.c | 2 +- examples/h5rados_example.h | 8 ++++++- examples/h5rados_file_create.c | 2 +- examples/h5rados_file_open.c | 2 +- examples/h5rados_group_create.c | 2 +- examples/h5rados_group_open.c | 2 +- examples/run_mobject_examples.sh | 48 ++++++++++++++++++++++++++++++++++++++++ 14 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 examples/run_mobject_examples.sh diff --git a/BRANCH.txt b/BRANCH.txt index 57aa2da..a0605bf 100644 --- a/BRANCH.txt +++ b/BRANCH.txt @@ -16,6 +16,8 @@ made to keep the MANIFEST sane, etc. * BUILDING * ************ +RADOS: + The code for the RADOS VOL connector lies entirely in the H5VLrados* files in the src directory. The connector is built as a static part of the library. @@ -30,6 +32,21 @@ LDFLAGS="-L/path/to/librados -lrados" YMMV, but the autotools do no librados hunting so you may need to specify things it can't figure out on its own. +MOBJECT: + +You'll need to be able to find the headers and libs for +mobject-store. You do not need the RADOS libraries. + +On jelly (THG), this will work: + +export SPACK_ROOT=/mnt/hdf/jsoumagne/spack +source $SPACK_ROOT/share/spack/setup-env.sh +module load GCC +source <(spack module tcl loads --dependencies mobject) + +You will also need to define HDF5_USE_MOBJECT in CPPFLAGS. +This switches between the RADOS and mobject headers. + *********** * TESTING * @@ -39,6 +56,8 @@ There are no tests for the RADOS VOL connector in the main library, so running 'make check' won't provide much information. +RADOS: + Once the library has been built, you can run some smoke checks by running 'make install' and going to the share/hdf5_examples/c/ directory and running the @@ -48,3 +67,21 @@ and run all the example scripts in a sane order. *** YOU WILL NEED TO COPY YOUR ceph.conf FILE *** INTO THIS DIRECTORY FIRST!!! + +In case it's not obvious, you'll need an active Ceph +cluster to run the tests :) + +MOBJECT: + +Same as above, only the script's name is run_mobject_examples.sh. + +You can probably ignore any sdskv_put warnings as long as the +examples pass. + +The tests are hard-coded to use /tmp/mobject-clust-test.gid and +this can be changed in h5rados_example.h. + +So to start mobject-server: + +mobject-server-daemon na+sm /tmp/mobject-cluster-test.gid + diff --git a/examples/Makefile.am b/examples/Makefile.am index ae0ac65..7e56f86 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -64,7 +64,7 @@ INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c \ h5rados_group_open.c h5rados_dset_create.c \ h5rados_dset_open.c h5rados_dset_write.c h5rados_dset_read.c \ h5rados_dset_wpartial.c h5rados_dset_rpartial.c \ - h5rados_example.h run_rados_examples.sh + h5rados_example.h run_rados_examples.sh run_mobject_examples.sh diff --git a/examples/h5rados_dset_create.c b/examples/h5rados_dset_create.c index 0e14a7f..2ec6bfa 100644 --- a/examples/h5rados_dset_create.c +++ b/examples/h5rados_dset_create.c @@ -14,7 +14,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_dset_open.c b/examples/h5rados_dset_open.c index 19e29ed..a0d4fa7 100644 --- a/examples/h5rados_dset_open.c +++ b/examples/h5rados_dset_open.c @@ -15,7 +15,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_dset_read.c b/examples/h5rados_dset_read.c index fc7292c..5dbb91d 100644 --- a/examples/h5rados_dset_read.c +++ b/examples/h5rados_dset_read.c @@ -18,7 +18,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_dset_rpartial.c b/examples/h5rados_dset_rpartial.c index 622b961..f6faad7 100644 --- a/examples/h5rados_dset_rpartial.c +++ b/examples/h5rados_dset_rpartial.c @@ -28,7 +28,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_dset_wpartial.c b/examples/h5rados_dset_wpartial.c index bb53982..048625e 100644 --- a/examples/h5rados_dset_wpartial.c +++ b/examples/h5rados_dset_wpartial.c @@ -28,7 +28,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_dset_write.c b/examples/h5rados_dset_write.c index dad96fa..17e6898 100644 --- a/examples/h5rados_dset_write.c +++ b/examples/h5rados_dset_write.c @@ -18,7 +18,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_example.h b/examples/h5rados_example.h index 4516b15..8da9487 100644 --- a/examples/h5rados_example.h +++ b/examples/h5rados_example.h @@ -3,7 +3,6 @@ #include #include #include -#include #include /* Macros for printing standard messages and issuing errors */ @@ -12,3 +11,10 @@ #define ERROR do {FAILED(); AT(); goto error;} while(0) #define PRINTF_ERROR(...) do {FAILED(); AT(); printf(" " __VA_ARGS__); printf("\n"); goto error;} while(0) +/* Config file */ +#ifdef HDF5_USE_MOBJECT +#define CEPH_CONFIG_FILE "/tmp/mobject-cluster-test.gid" +#else +#define CEPH_CONFIG_FILE "ceph.conf" +#endif + diff --git a/examples/h5rados_file_create.c b/examples/h5rados_file_create.c index 493be18..04101b3 100644 --- a/examples/h5rados_file_create.c +++ b/examples/h5rados_file_create.c @@ -12,7 +12,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_file_open.c b/examples/h5rados_file_open.c index 176d26e..881ab10 100644 --- a/examples/h5rados_file_open.c +++ b/examples/h5rados_file_open.c @@ -12,7 +12,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_group_create.c b/examples/h5rados_group_create.c index fcfbbc0..68c6a6d 100644 --- a/examples/h5rados_group_create.c +++ b/examples/h5rados_group_create.c @@ -12,7 +12,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/h5rados_group_open.c b/examples/h5rados_group_open.c index 3339480..4fde06c 100644 --- a/examples/h5rados_group_open.c +++ b/examples/h5rados_group_open.c @@ -12,7 +12,7 @@ int main(int argc, char *argv[]) { if(rados_create(&cluster, NULL) < 0) ERROR; - if(rados_conf_read_file(cluster, "ceph.conf") < 0) + if(rados_conf_read_file(cluster, CEPH_CONFIG_FILE) < 0) ERROR; /* Initialize VOL */ diff --git a/examples/run_mobject_examples.sh b/examples/run_mobject_examples.sh new file mode 100644 index 0000000..767387d --- /dev/null +++ b/examples/run_mobject_examples.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Build mobject examples. +# Assumes mobject stuff can be found +echo "Building mobject examples" +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_file_create h5rados_file_create.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_file_open h5rados_file_open.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_dset_create h5rados_dset_create.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_dset_open h5rados_dset_open.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_group_create h5rados_group_create.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_group_open h5rados_group_open.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_dset_write h5rados_dset_write.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_dset_read h5rados_dset_read.c + +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_dset_rpartial h5rados_dset_rpartial.c +../../../bin/h5pcc -DHDF5_USE_MOBJECT -o h5rados_dset_wpartial h5rados_dset_wpartial.c +echo "DONE" +echo + +# Run the mobject examples +echo "Running mobject examples" +# File create/open +echo "./h5rados_file_create testfile" +mpiexec -n 2 ./h5rados_file_create testfile +echo "./h5rados_file_open testfile" +mpiexec -n 2 ./h5rados_file_open testfile +# Dataset create/open +echo "./h5rados_dset_create testfile testdset" +mpiexec -n 2 ./h5rados_dset_create testfile testdset +echo "./h5rados_dset_open testfile testdset" +mpiexec -n 2 ./h5rados_dset_open testfile testdset +# Group create/open +echo "./h5rados_group_create testfile testgroup" +mpiexec -n 2 ./h5rados_group_create testfile testgroup +echo "./h5rados_group_open testfile testgroup" +mpiexec -n 2 ./h5rados_group_open testfile testgroup +# Dataset read/write +echo "./h5rados_dset_write testfile testdset" +mpiexec -n 2 ./h5rados_dset_write testfile testdset +echo "./h5rados_dset_read testfile testdset" +mpiexec -n 2 ./h5rados_dset_read testfile testdset +# Dataset read/write (partial) +echo "./h5rados_dset_wpartial testfile testdset" +mpiexec -n 2 ./h5rados_dset_wpartial testfile testdset +echo "./h5rados_dset_rpartial testfile testdset" +mpiexec -n 2 ./h5rados_dset_rpartial testfile testdset +echo + -- cgit v0.12