From 9752f525152eab4d959f109797ffd941d438d62f Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Fri, 18 Nov 2016 17:16:43 -0600 Subject: Improve transaction support, add H5Fcreate_ff, H5Fopen_ff, H5Fclose_ff. --- src/H5FF.c | 278 ++++++++++++++++++++++++++++++++++++++++++++++++++---- src/H5FFprivate.h | 4 + src/H5FFpublic.h | 16 ++-- src/H5Pfapl.c | 7 ++ src/H5TR.c | 2 + src/H5VLdaosm.c | 226 ++++++++++++++++++++++++++++++-------------- 6 files changed, 436 insertions(+), 97 deletions(-) diff --git a/src/H5FF.c b/src/H5FF.c index b65b200..649d84f 100644 --- a/src/H5FF.c +++ b/src/H5FF.c @@ -24,11 +24,11 @@ #include "H5FFmodule.h" /* This source code file is part of the H5FF module */ -#define H5A_FRIEND /*suppress error about including H5Apkg */ +//#define H5A_FRIEND /*suppress error about including H5Apkg */ #define H5D_FRIEND /*suppress error about including H5Dpkg */ #define H5F_FRIEND /*suppress error about including H5Fpkg */ -#define H5G_FRIEND /*suppress error about including H5Gpkg */ -#define H5T_FRIEND /*suppress error about including H5Tpkg */ +//#define H5G_FRIEND /*suppress error about including H5Gpkg */ +//#define H5T_FRIEND /*suppress error about including H5Tpkg */ /***********/ @@ -45,6 +45,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ //#include "H5Tpkg.h" /* Datatype access */ +#include "H5TRprivate.h" /* Transactions */ #include "H5VLdaosm.h" /* IOD plugin - tmp */ @@ -52,7 +53,8 @@ /****************/ /* Local Macros */ /****************/ -H5FL_EXTERN(H5RC_t); +H5FL_EXTERN(H5TR_t); +H5FL_EXTERN(H5VL_t); /******************/ /* Local Typedefs */ @@ -91,25 +93,272 @@ H5FF__init_package(void) HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init file interface") /*if(H5G_init() < 0) - HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init group interface") + HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init group interface")*/ if(H5D_init() < 0) HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init dataset interface") - if(H5A_init() < 0) + /*if(H5A_init() < 0) HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init attribute interface") if(H5M_init() < 0) - HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init map interface") - - if(H5RC_init() < 0) HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init map interface") DSMINC*/ + if(H5TR_init() < 0) + HDONE_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to init map interface") + FUNC_LEAVE_NOAPI(ret_value) } /* H5FF__init_package() */ /*------------------------------------------------------------------------- + * Function: H5Fcreate_ff + * + * Purpose: Asynchronous wrapper around H5Fcreate(). If requested, + * trans_id returns the transaction used to create the file, + * uncommitted. If trans_id is NULL, the transaction used to + * create the file will be committed. + * + * Return: Success: The placeholder ID for a new file. When + * the asynchronous operation completes, this + * ID will transparently be modified to be a + * "normal" ID. + * Failure: FAIL + * + * Programmer: Neil Fortner + * Monday, November 14, 2016 + * + *------------------------------------------------------------------------- + */ +hid_t +H5Fcreate_ff(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id, + hid_t *trans_id) +{ + void *file = NULL; /* file token from VOL plugin */ + H5P_genplist_t *plist; /* Property list pointer */ + H5VL_plugin_prop_t plugin_prop; /* Property for vol plugin ID & info */ + H5VL_class_t *vol_cls = NULL; /* VOL Class structure for callback info */ + H5VL_t *vol_info = NULL; /* VOL info struct */ + hid_t ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check/fix arguments */ + if(!filename || !*filename) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file name") + /* In this routine, we only accept the following flags: + * H5F_ACC_EXCL, H5F_ACC_TRUNC and H5F_ACC_DEBUG + */ + if(flags & ~(H5F_ACC_EXCL | H5F_ACC_TRUNC | H5F_ACC_DEBUG)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags") + /* The H5F_ACC_EXCL and H5F_ACC_TRUNC flags are mutually exclusive */ + if((flags & H5F_ACC_EXCL) && (flags & H5F_ACC_TRUNC)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "mutually exclusive flags for file creation") + + /* Check file creation property list */ + if(H5P_DEFAULT == fcpl_id) + fcpl_id = H5P_FILE_CREATE_DEFAULT; + else + if(TRUE != H5P_isa_class(fcpl_id, H5P_FILE_CREATE)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not file create property list") + + /* Check the file access property list */ + if(H5P_DEFAULT == fapl_id) + fapl_id = H5P_FILE_ACCESS_DEFAULT; + else + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not file access property list") + + /* get the VOL info from the fapl */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + + if(H5P_peek(plist, H5F_ACS_VOL_NAME, &plugin_prop) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get vol plugin info") + + if(NULL == (vol_cls = (H5VL_class_t *)H5I_object_verify(plugin_prop.plugin_id, H5I_VOL))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a VOL plugin ID") + + /* Determine if we want to acquire the transaction used to create the file + */ + if(trans_id) { + H5TR_t *tr = NULL; + + /* Allocate transaction struct */ + if(NULL == (tr = H5FL_CALLOC(H5TR_t))) + HGOTO_ERROR(H5E_SYM, H5E_NOSPACE, FAIL, "can't allocate transaction structure") + + tr->file = NULL; + tr->epoch = DAOS_EPOCH_MAX; + tr->vol_cls = vol_cls; + + /* Get an atom for the transaction */ + if((*trans_id = H5I_register(H5I_TR, tr, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize transaction handle"); + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + if(H5P_set(plist, H5VL_ACQUIRE_TR_ID, trans_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set property value for trans id") + } /* end if */ + + /* create a new file or truncate an existing file through the VOL */ + if(NULL == (file = H5VL_file_create(vol_cls, filename, flags, fcpl_id, fapl_id, + H5AC_ind_read_dxpl_id, NULL))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create file") + + /* setup VOL info struct */ + if(NULL == (vol_info = H5FL_CALLOC(H5VL_t))) + HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, FAIL, "can't allocate VL info struct") + vol_info->vol_cls = vol_cls; + vol_info->vol_id = plugin_prop.plugin_id; + if(H5I_inc_ref(vol_info->vol_id, FALSE) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTINC, FAIL, "unable to increment ref count on VOL plugin") + + /* Get an atom for the file */ + if((ret_value = H5VL_register_id(H5I_FILE, file, vol_info, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Fcreate_ff() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fopen_ff + * + * Purpose: Asynchronous wrapper around H5Fcreate(). If requested, + * trans_id returns the highest committed transaction for the + * file. + * + * Return: Success: The placeholder ID for a new file. When + * the asynchronous operation completes, this + * ID will transparently be modified to be a + * "normal" ID. + * Failure: FAIL + * + * Programmer: Neil Fortner + * Monday, November 14, 2016 + * + *------------------------------------------------------------------------- + */ +hid_t +H5Fopen_ff(const char *filename, unsigned flags, hid_t fapl_id, hid_t *trans_id) +{ + void *file = NULL; /* file token from VOL plugin */ + H5P_genplist_t *plist; /* Property list pointer */ + H5VL_plugin_prop_t plugin_prop; /* Property for vol plugin ID & info */ + H5VL_class_t *vol_cls = NULL; /* VOL Class structure for callback info */ + H5VL_t *vol_info = NULL; /* VOL info struct */ + hid_t ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + + /* Check/fix arguments */ + if(!filename || !*filename) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file name") + /* Reject undefined flags (~H5F_ACC_PUBLIC_FLAGS) and the H5F_ACC_TRUNC & H5F_ACC_EXCL flags */ + if((flags & ~H5F_ACC_PUBLIC_FLAGS) || + (flags & H5F_ACC_TRUNC) || (flags & H5F_ACC_EXCL)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file open flags") + + /* Check the file access property list */ + if(H5P_DEFAULT == fapl_id) + fapl_id = H5P_FILE_ACCESS_DEFAULT; + else + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not file access property list") + + /* get the VOL info from the fapl */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + + if(H5P_peek(plist, H5F_ACS_VOL_NAME, &plugin_prop) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get vol plugin info") + + if(NULL == (vol_cls = (H5VL_class_t *)H5I_object_verify(plugin_prop.plugin_id, H5I_VOL))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a VOL plugin ID") + + /* Determine if we want to acquire the highest committed transaction when + * the file is opened */ + if(trans_id) { + H5TR_t *tr = NULL; + + /* Allocate transaction struct */ + if(NULL == (tr = H5FL_CALLOC(H5TR_t))) + HGOTO_ERROR(H5E_SYM, H5E_NOSPACE, FAIL, "can't allocate transaction structure") + + tr->file = NULL; + tr->epoch = DAOS_EPOCH_MAX; + tr->vol_cls = vol_cls; + + /* Get an atom for the transaction */ + if((*trans_id = H5I_register(H5I_TR, tr, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize transaction handle"); + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + if(H5P_set(plist, H5VL_ACQUIRE_TR_ID, trans_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set property value for trans id") + } /* end if */ + + /* create a new file or truncate an existing file through the VOL */ + if(NULL == (file = H5VL_file_open(vol_cls, filename, flags, fapl_id, + H5AC_ind_read_dxpl_id, NULL))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to open file") + + /* setup VOL info struct */ + if(NULL == (vol_info = H5FL_CALLOC(H5VL_t))) + HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, FAIL, "can't allocate VL info struct") + vol_info->vol_cls = vol_cls; + vol_info->vol_id = plugin_prop.plugin_id; + if(H5I_inc_ref(vol_info->vol_id, FALSE) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTINC, FAIL, "unable to increment ref count on VOL plugin") + + /* Get an atom for the file */ + if((ret_value = H5VL_register_id(H5I_FILE, file, vol_info, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Fopen_ff() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fclose_ff + * + * Purpose: This function closes the file specified by FILE_ID, + * terminating access to the file through FILE_ID. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Neil Fortner + * November 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fclose_ff(hid_t file_id, hid_t H5_ATTR_UNUSED trans_id) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + + /* flush file using trans_id? DSMINC */ + + /* Decrement reference count on atom. When it reaches zero the file will be closed. */ + if(H5I_dec_app_ref(file_id) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTCLOSEFILE, FAIL, "decrementing file ID failed") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Fclose_ff */ + + +/*------------------------------------------------------------------------- * Function: H5Dcreate_ff * * Purpose: Asynchronous wrapper around H5Dcreate(). @@ -127,8 +376,7 @@ H5FF__init_package(void) */ hid_t H5Dcreate_ff(hid_t loc_id, const char *name, hid_t type_id, hid_t space_id, - hid_t lcpl_id, hid_t dcpl_id, hid_t dapl_id, hid_t trans_id, - hid_t H5_ATTR_UNUSED estack_id) + hid_t lcpl_id, hid_t dcpl_id, hid_t dapl_id, hid_t trans_id) { void *dset = NULL; /* dset token from VOL plugin */ H5VL_object_t *obj = NULL; /* object token of loc_id */ @@ -138,8 +386,6 @@ H5Dcreate_ff(hid_t loc_id, const char *name, hid_t type_id, hid_t space_id, hid_t ret_value; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE9("i", "i*siiiiiii", loc_id, name, type_id, space_id, lcpl_id, dcpl_id, - dapl_id, trans_id, estack_id); if(!name || !*name) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no name") @@ -224,7 +470,7 @@ done: *------------------------------------------------------------------------- */ hid_t -H5Dopen_ff(hid_t loc_id, const char *name, hid_t dapl_id, hid_t trans_id, hid_t H5_ATTR_UNUSED estack_id) +H5Dopen_ff(hid_t loc_id, const char *name, hid_t dapl_id, hid_t trans_id) { void *dset = NULL; /* dset token from VOL plugin */ H5VL_object_t *obj = NULL; /* object token of loc_id */ @@ -234,7 +480,6 @@ H5Dopen_ff(hid_t loc_id, const char *name, hid_t dapl_id, hid_t trans_id, hid_t hid_t ret_value; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE5("i", "i*siii", loc_id, name, dapl_id, trans_id, estack_id); /* Check args */ if(!name || !*name) @@ -292,13 +537,12 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Dclose_ff(hid_t dset_id, hid_t H5_ATTR_UNUSED estack_id) +H5Dclose_ff(hid_t dset_id, hid_t H5_ATTR_UNUSED trans_id) { H5VL_object_t *dset; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE2("e", "ii", dset_id, estack_id); /* Check args */ if(NULL == (dset = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) diff --git a/src/H5FFprivate.h b/src/H5FFprivate.h index cfbb49e..7345b80 100755 --- a/src/H5FFprivate.h +++ b/src/H5FFprivate.h @@ -32,6 +32,10 @@ /* DXPL property to store the transaction ID from the FF wrappers */ #define H5VL_TRANS_ID "transaction_id" +/* property to tell the VOL plugin to acquire a transaction on the container + * when the file is created or opened */ +#define H5VL_ACQUIRE_TR_ID "acquire_transaction" + /****************************/ /* Library Private Typedefs */ /****************************/ diff --git a/src/H5FFpublic.h b/src/H5FFpublic.h index b6585ae..4ccdb01 100644 --- a/src/H5FFpublic.h +++ b/src/H5FFpublic.h @@ -49,17 +49,17 @@ extern "C" { /*********************/ /* API wrappers */ -/*H5_DLL hid_t H5Fcreate_ff(const char *filename, unsigned flags, hid_t fcpl, - hid_t fapl, hid_t estack_id); +H5_DLL hid_t H5Fcreate_ff(const char *filename, unsigned flags, hid_t fcpl_id, + hid_t fapl_id, hid_t *trans_id); H5_DLL hid_t H5Fopen_ff(const char *filename, unsigned flags, hid_t fapl_id, - hid_t *rcxt_id, hid_t estack_id); -H5_DLL herr_t H5Fclose_ff(hid_t file_id, hbool_t persist_flag, hid_t estack_id);*/ + hid_t *trans_id); +H5_DLL herr_t H5Fclose_ff(hid_t file_id, hid_t trans_id); H5_DLL hid_t H5Dcreate_ff(hid_t loc_id, const char *name, hid_t type_id, - hid_t space_id, hid_t lcpl_id, hid_t dcpl_id, hid_t dapl_id, hid_t trans_id, - hid_t estack_id); + hid_t space_id, hid_t lcpl_id, hid_t dcpl_id, hid_t dapl_id, + hid_t trans_id); H5_DLL hid_t H5Dopen_ff(hid_t loc_id, const char *name, hid_t dapl_id, - hid_t trans_id, hid_t estack_id); -H5_DLL herr_t H5Dclose_ff(hid_t dset_id, hid_t estack_id); + hid_t trans_id); +H5_DLL herr_t H5Dclose_ff(hid_t dset_id, hid_t trans_id); #endif /* H5_HAVE_EFF */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 61a63ea..e8bc054 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -40,6 +40,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* Files */ #include "H5FDprivate.h" /* File drivers */ +#include "H5FFprivate.h" /* FFwd wrappers */ #include "H5VLprivate.h" /* VOL plugins */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory Management */ @@ -342,6 +343,7 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) const H5FD_driver_prop_t def_driver_prop = H5F_ACS_FILE_DRV_DEF; /* Default VFL driver ID & info (initialized from a variable) */ const H5VL_plugin_prop_t def_vol_prop = H5F_ACS_VOL_DEF; /* Default VOL plugin ID & info (initialized from a variable) */ + hid_t trans_id = FAIL; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -488,6 +490,11 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the transaction ID acquire property*/ + if(H5P_register_real(pclass, H5VL_ACQUIRE_TR_ID, sizeof(hid_t), &trans_id, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + #ifdef H5_HAVE_PARALLEL /* Register the metadata collective read flag */ if(H5P_register_real(pclass, H5_COLL_MD_READ_FLAG_NAME, H5F_ACS_COLL_MD_READ_FLAG_SIZE, &H5F_def_coll_md_read_flag_g, diff --git a/src/H5TR.c b/src/H5TR.c index cfee25f..51323cc 100644 --- a/src/H5TR.c +++ b/src/H5TR.c @@ -373,6 +373,8 @@ H5TR_close(H5TR_t *tr) FUNC_ENTER_NOAPI_NOINIT_NOERR + /*TODO: Keep track of lowest transaction number (LRE), slip daos epoch as it + * moves forward. */ tr = H5FL_FREE(H5TR_t, tr); FUNC_LEAVE_NOAPI(SUCCEED) diff --git a/src/H5VLdaosm.c b/src/H5VLdaosm.c index c79d995..3495df5 100644 --- a/src/H5VLdaosm.c +++ b/src/H5VLdaosm.c @@ -507,10 +507,11 @@ H5VL_daosm_file_create(const char *name, unsigned flags, hid_t fcpl_id, H5P_genplist_t *plist = NULL; /* Property list pointer */ H5VL_daosm_file_t *file = NULL; daos_epoch_t epoch; + hid_t trans_id; daos_iov_t glob; - uint64_t gh_sizes[2]; + uint64_t bcast_buf_64[6]; char *gh_buf = NULL; - daos_obj_id_t oid = {0, 0, 0}; + daos_obj_id_t gmd_oid = {0, 0, 0}; hbool_t must_bcast = FALSE; int ret; void *ret_value = NULL; @@ -552,6 +553,9 @@ H5VL_daosm_file_create(const char *name, unsigned flags, hid_t fcpl_id, /* Hash file name to create uuid */ H5VL_daosm_hash128(name, &file->uuid); + /* Generate oid for global metadata object */ + daos_obj_id_generate(&gmd_oid, DAOS_OC_REPLICA_RW); + if(file->my_rank == 0) { daos_epoch_state_t epoch_state; @@ -594,8 +598,7 @@ H5VL_daosm_file_create(const char *name, unsigned flags, hid_t fcpl_id, HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't hold epoch: %d", ret) /* Create global metadata object */ - daos_obj_id_generate(&oid, DAOS_OC_REPLICA_RW); - if(0 != (ret = daos_obj_declare(file->coh, oid, 0, NULL /*oa*/, NULL /*event*/))) + if(0 != (ret = daos_obj_declare(file->coh, gmd_oid, 0, NULL /*oa*/, NULL /*event*/))) HGOTO_ERROR(H5E_FILE, H5E_CANTCREATE, NULL, "can't create global metadata object: %d", ret) /* Create root group */ @@ -608,7 +611,8 @@ H5VL_daosm_file_create(const char *name, unsigned flags, hid_t fcpl_id, if(0 != (ret = daos_obj_open(file->coh, file->root_grp->oid, epoch, DAOS_OO_RW, &file->root_grp->obj_oh, NULL /*event*/))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "can't open root group: %d", ret) - /* Write root group OID to global metadata object DSMINC */ + /* Write root group OID, max oid per process to global metadata object + * DSMINC */ /* Bcast global handles if there are other processes */ if(file->num_procs > 1) { @@ -618,25 +622,34 @@ H5VL_daosm_file_create(const char *name, unsigned flags, hid_t fcpl_id, glob.iov_len = 0; if(0 != (ret = daos_pool_local2global(file->poh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global pool handle size: %d", ret) - gh_sizes[0] = (uint64_t)glob.iov_buf_len; + bcast_buf_64[0] = (uint64_t)glob.iov_buf_len; glob.iov_buf = NULL; glob.iov_buf_len = 0; glob.iov_len = 0; if(0 != (ret = daos_cont_local2global(file->coh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global container handle size: %d", ret) - gh_sizes[1] = (uint64_t)glob.iov_buf_len; + bcast_buf_64[1] = (uint64_t)glob.iov_buf_len; + + /* Add root group oid to bcast_buf_64 */ + bcast_buf_64[2] = file->root_grp->oid.lo; + bcast_buf_64[3] = file->root_grp->oid.mid; + bcast_buf_64[4] = file->root_grp->oid.hi; + + /* Add epoch to bcast_buf_64 */ + HDassert(sizeof(bcast_buf_64[5]) == sizeof(epoch)); + bcast_buf_64[5] = (uint64_t)epoch; /* Retrieve global pool and container handles */ - if(NULL == (gh_buf = (char *)H5MM_malloc(gh_sizes[0] + gh_sizes[1]))) + if(NULL == (gh_buf = (char *)H5MM_malloc(bcast_buf_64[0] + bcast_buf_64[1]))) HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for global handles") glob.iov_buf = gh_buf; - glob.iov_buf_len = gh_sizes[0]; + glob.iov_buf_len = bcast_buf_64[0]; glob.iov_len = 0; if(0 != (ret = daos_pool_local2global(file->poh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global pool handle: %d", ret) HDassert(glob.iov_len == glob.iov_buf_len); - glob.iov_buf = gh_buf + gh_sizes[0]; - glob.iov_buf_len = gh_sizes[1]; + glob.iov_buf = gh_buf + bcast_buf_64[0]; + glob.iov_buf_len = bcast_buf_64[1]; glob.iov_len = 0; if(0 != (ret = daos_cont_local2global(file->coh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global container handle: %d", ret) @@ -645,58 +658,64 @@ H5VL_daosm_file_create(const char *name, unsigned flags, hid_t fcpl_id, /* We are about to bcast so we no longer need to bcast on failure */ must_bcast = FALSE; - /* MPI_Bcast gh_sizes */ - if(MPI_SUCCESS != MPI_Bcast(gh_sizes, 2, MPI_UINT64_T, 0, fa->comm)) + /* MPI_Bcast bcast_buf_64 */ + if(MPI_SUCCESS != MPI_Bcast(bcast_buf_64, sizeof(bcast_buf_64)/sizeof(bcast_buf_64[0]), MPI_UINT64_T, 0, fa->comm)) HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") /* MPI_Bcast gh_buf */ - if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(gh_sizes[0] + gh_sizes[1]), MPI_BYTE, 0, fa->comm)) + if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(bcast_buf_64[0] + bcast_buf_64[1]), MPI_BYTE, 0, fa->comm)) HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") } /* end if */ - - /* Commit epoch DSMINC */ - if(0 != (ret = daos_epoch_commit(file->coh, epoch, NULL /*state*/, NULL /*event*/))) - HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, NULL, "can't commit epoch: %d", ret) - epoch++; } /* end if */ else { - /* Receive gh_sizes */ - if(MPI_SUCCESS != MPI_Bcast(gh_sizes, 2, MPI_UINT64_T, 0, fa->comm)) + /* Receive bcast_buf_64 */ + if(MPI_SUCCESS != MPI_Bcast(bcast_buf_64, sizeof(bcast_buf_64)/sizeof(bcast_buf_64[0]), MPI_UINT64_T, 0, fa->comm)) HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") - /* Check for gh_sizes set to 0 - indicates failure */ - if(gh_sizes[0] == 0) { - HDassert(gh_sizes[1] == 0); + /* Check for bcast_buf_64[0] set to 0 - indicates failure */ + if(bcast_buf_64[0] == 0) { + HDassert(bcast_buf_64[1] == 0); HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "lead process failed to open file") } /* end if */ + /* Retrieve root group oid from bcast_buf_64 */ + file->root_grp->oid.lo = bcast_buf_64[2]; + file->root_grp->oid.mid = bcast_buf_64[3]; + file->root_grp->oid.hi = bcast_buf_64[4]; + + /* Retrieve epoch from bcast_buf_64 */ + HDassert(sizeof(bcast_buf_64[5]) == sizeof(epoch)); + epoch = (daos_epoch_t)bcast_buf_64[5]; + /* Allocate global handle buffer */ - if(NULL == (gh_buf = (char *)H5MM_malloc(gh_sizes[0] + gh_sizes[1]))) + if(NULL == (gh_buf = (char *)H5MM_malloc(bcast_buf_64[0] + bcast_buf_64[1]))) HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for global handles") /* Receive gh_buf */ - if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(gh_sizes[0] + gh_sizes[1]), MPI_BYTE, 0, fa->comm)) + if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(bcast_buf_64[0] + bcast_buf_64[1]), MPI_BYTE, 0, fa->comm)) HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") /* Create local pool and container handles */ glob.iov_buf = gh_buf; - glob.iov_buf_len = gh_sizes[0]; - glob.iov_len = gh_sizes[0]; + glob.iov_buf_len = bcast_buf_64[0]; + glob.iov_len = bcast_buf_64[0]; if(0 != (ret = daos_pool_global2local(glob, &file->poh))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "can't get local pool handle: %d", ret) - glob.iov_buf = gh_buf + gh_sizes[0]; - glob.iov_buf_len = gh_sizes[1]; - glob.iov_len = gh_sizes[1]; + glob.iov_buf = gh_buf + bcast_buf_64[0]; + glob.iov_buf_len = bcast_buf_64[1]; + glob.iov_len = bcast_buf_64[1]; if(0 != (ret = daos_cont_global2local(file->poh, glob, &file->coh))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "can't get local container handle: %d", ret) - /* Leave root group handle empty for now */ + /* Open root group */ + if(0 != (ret = daos_obj_open(file->coh, file->root_grp->oid, epoch, DAOS_OO_RW, &file->root_grp->obj_oh, NULL /*event*/))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "can't open root group: %d", ret) /* Handle pool_info and container_info DSMINC */ } /* end else */ /* Open global metadata object */ - if(0 != (ret = daos_obj_open(file->glob_md_oh, oid, 0, DAOS_OO_RW, &file->glob_md_oh, NULL /*event*/))) + if(0 != (ret = daos_obj_open(file->glob_md_oh, gmd_oid, 0, DAOS_OO_RW, &file->glob_md_oh, NULL /*event*/))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "can't open global metadata object: %d", ret) /* Finish setting up file struct */ @@ -718,6 +737,29 @@ H5VL_daosm_file_create(const char *name, unsigned flags, hid_t fcpl_id, if(FAIL == H5FD_mpi_comm_info_dup(fa->comm, fa->info, &file->comm, &file->info)) HGOTO_ERROR(H5E_INTERNAL, H5E_CANTCOPY, NULL, "Communicator/Info duplicate failed"); + /* Determine if we want to acquire a transaction for the file creation */ + if(H5P_get(plist, H5VL_ACQUIRE_TR_ID, &trans_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get property value for trans id"); + + if(FAIL == trans_id) { + if(file->my_rank == 0) + /* Transaction not requested, commit the epoch */ + if(0 != (ret = daos_epoch_commit(file->coh, epoch, NULL /*state*/, NULL /*event*/))) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, NULL, "can't commit epoch: %d", ret) + } /* end if */ + else { + /* Transaction requested, do not commit the epoch - it is up to the + * application */ + H5TR_t *tr = NULL; + + /* get the TR object */ + if(NULL == (tr = (H5TR_t *)H5I_object_verify(trans_id, H5I_TR))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "not a TRANSACTION ID"); + + tr->epoch = epoch; + tr->file = file; + } /* end else */ + ret_value = (void *)file; done: @@ -727,12 +769,11 @@ done: /* If the operation is synchronous and it failed at the server, or it failed * locally, then cleanup and return fail */ if(NULL == ret_value) { - /* Bcast gh_sizes as '0' if necessary - this will trigger failures in - * the other processes so we do not need to do the second bcast. */ + /* Bcast bcast_buf_64 as '0' if necessary - this will trigger failures + * in the other processes so we do not need to do the second bcast. */ if(must_bcast) { - gh_sizes[0] = 0; - gh_sizes[1] = 0; - if(MPI_SUCCESS != MPI_Bcast(gh_sizes, 2, MPI_UINT64_T, 0, fa->comm)) + HDmemset(bcast_buf_64, 0, sizeof(bcast_buf_64)); + if(MPI_SUCCESS != MPI_Bcast(bcast_buf_64, sizeof(bcast_buf_64)/sizeof(bcast_buf_64[0]), MPI_UINT64_T, 0, fa->comm)) HDONE_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") } /* end if */ @@ -766,10 +807,11 @@ H5VL_daosm_file_open(const char *name, unsigned flags, hid_t fapl_id, H5P_genplist_t *plist = NULL; /* Property list pointer */ H5VL_daosm_file_t *file = NULL; daos_epoch_t epoch; + hid_t trans_id; daos_iov_t glob; - uint64_t gh_sizes[2]; + uint64_t bcast_buf_64[6]; char *gh_buf = NULL; - daos_obj_id_t oid = {0, 0, 0}; + daos_obj_id_t gmd_oid = {0, 0, 0}; hbool_t must_bcast = FALSE; int ret; void *ret_value = NULL; @@ -802,6 +844,9 @@ H5VL_daosm_file_open(const char *name, unsigned flags, hid_t fapl_id, /* Hash file name to create uuid */ H5VL_daosm_hash128(name, &file->uuid); + /* Generate oid for global metadata object */ + daos_obj_id_generate(&gmd_oid, DAOS_OC_REPLICA_RW); + if(file->my_rank == 0) { daos_epoch_state_t epoch_state; @@ -827,6 +872,9 @@ H5VL_daosm_file_open(const char *name, unsigned flags, hid_t fapl_id, if(0 != (ret = daos_epoch_hold(file->coh, &epoch, NULL /*state*/, NULL /*event*/))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't hold epoch: %d", ret) + /* Read from the HCE */ + epoch--; + /* Open root group */ /* Read root group OID from global metadata object DSMINC */ file->root_grp->oid.lo = 1; @@ -842,25 +890,34 @@ H5VL_daosm_file_open(const char *name, unsigned flags, hid_t fapl_id, glob.iov_len = 0; if(0 != (ret = daos_pool_local2global(file->poh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global pool handle size: %d", ret) - gh_sizes[0] = (uint64_t)glob.iov_buf_len; + bcast_buf_64[0] = (uint64_t)glob.iov_buf_len; glob.iov_buf = NULL; glob.iov_buf_len = 0; glob.iov_len = 0; if(0 != (ret = daos_cont_local2global(file->coh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global container handle size: %d", ret) - gh_sizes[1] = (uint64_t)glob.iov_buf_len; + bcast_buf_64[1] = (uint64_t)glob.iov_buf_len; + + /* Add root group oid to bcast_buf_64 */ + bcast_buf_64[2] = file->root_grp->oid.lo; + bcast_buf_64[3] = file->root_grp->oid.mid; + bcast_buf_64[4] = file->root_grp->oid.hi; + + /* Add epoch to bcast_buf_64 */ + HDassert(sizeof(bcast_buf_64[5]) == sizeof(epoch)); + bcast_buf_64[5] = (uint64_t)epoch; /* Retrieve global pool and container handles */ - if(NULL == (gh_buf = (char *)malloc(gh_sizes[0] + gh_sizes[1]))) - HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for global handles") + if(NULL == (gh_buf = (char *)malloc(bcast_buf_64[0] + bcast_buf_64[1]))) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, bcast_buf_64, "can't allocate space for global handles") glob.iov_buf = gh_buf; - glob.iov_buf_len = gh_sizes[0]; + glob.iov_buf_len = bcast_buf_64[0]; glob.iov_len = 0; if(0 != (ret = daos_pool_local2global(file->poh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global pool handle: %d", ret) HDassert(glob.iov_len == glob.iov_buf_len); - glob.iov_buf = gh_buf + gh_sizes[0]; - glob.iov_buf_len = gh_sizes[1]; + glob.iov_buf = gh_buf + bcast_buf_64[0]; + glob.iov_buf_len = bcast_buf_64[1]; glob.iov_len = 0; if(0 != (ret = daos_cont_local2global(file->coh, &glob))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't get global container handle: %d", ret) @@ -869,54 +926,64 @@ H5VL_daosm_file_open(const char *name, unsigned flags, hid_t fapl_id, /* We are about to bcast so we no longer need to bcast on failure */ must_bcast = FALSE; - /* MPI_Bcast gh_sizes */ - if(MPI_SUCCESS != MPI_Bcast(gh_sizes, 2, MPI_UINT64_T, 0, fa->comm)) + /* MPI_Bcast bcast_buf_64 */ + if(MPI_SUCCESS != MPI_Bcast(bcast_buf_64, sizeof(bcast_buf_64)/sizeof(bcast_buf_64[0]), MPI_UINT64_T, 0, fa->comm)) HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") /* MPI_Bcast gh_buf */ - if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(gh_sizes[0] + gh_sizes[1]), MPI_BYTE, 0, fa->comm)) - HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") + if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(bcast_buf_64[0] + bcast_buf_64[1]), MPI_BYTE, 0, fa->comm)) + HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handles") } /* end if */ } /* end if */ else { - /* Receive gh_sizes */ - if(MPI_SUCCESS != MPI_Bcast(gh_sizes, 2, MPI_UINT64_T, 0, fa->comm)) + /* Receive bcast_buf_64 */ + if(MPI_SUCCESS != MPI_Bcast(bcast_buf_64, sizeof(bcast_buf_64)/sizeof(bcast_buf_64[0]), MPI_UINT64_T, 0, fa->comm)) HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") - /* Check for gh_sizes set to 0 - indicates failure */ - if(gh_sizes[0] == 0) { - HDassert(gh_sizes[1] == 0); + /* Check for bcast_buf_64[0] set to 0 - indicates failure */ + if(bcast_buf_64[0] == 0) { + HDassert(bcast_buf_64[1] == 0); HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "lead process failed to open file") } /* end if */ + /* Retrieve root group oid from bcast_buf_64 */ + file->root_grp->oid.lo = bcast_buf_64[2]; + file->root_grp->oid.mid = bcast_buf_64[3]; + file->root_grp->oid.hi = bcast_buf_64[4]; + + /* Retrieve epoch from bcast_buf_64 */ + HDassert(sizeof(bcast_buf_64[5]) == sizeof(epoch)); + epoch = (daos_epoch_t)bcast_buf_64[5]; + /* Allocate global handle buffer */ - if(NULL == (gh_buf = (char *)malloc(gh_sizes[0] + gh_sizes[1]))) + if(NULL == (gh_buf = (char *)malloc(bcast_buf_64[0] + bcast_buf_64[1]))) HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for global handles") /* Receive gh_buf */ - if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(gh_sizes[0] + gh_sizes[1]), MPI_BYTE, 0, fa->comm)) - HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") + if(MPI_SUCCESS != MPI_Bcast(gh_buf, (int)(bcast_buf_64[0] + bcast_buf_64[1]), MPI_BYTE, 0, fa->comm)) + HGOTO_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handles") /* Create local pool and container handles */ glob.iov_buf = gh_buf; - glob.iov_buf_len = gh_sizes[0]; - glob.iov_len = gh_sizes[0]; + glob.iov_buf_len = bcast_buf_64[0]; + glob.iov_len = bcast_buf_64[0]; if(0 != (ret = daos_pool_global2local(glob, &file->poh))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "can't get local pool handle: %d", ret) - glob.iov_buf = gh_buf + gh_sizes[0]; - glob.iov_buf_len = gh_sizes[1]; - glob.iov_len = gh_sizes[1]; + glob.iov_buf = gh_buf + bcast_buf_64[0]; + glob.iov_buf_len = bcast_buf_64[1]; + glob.iov_len = bcast_buf_64[1]; if(0 != (ret = daos_cont_global2local(file->poh, glob, &file->coh))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "can't get local container handle: %d", ret) - /* Leave root group handle empty for now */ + /* Open root group */ + if(0 != (ret = daos_obj_open(file->coh, file->root_grp->oid, epoch, DAOS_OO_RW, &file->root_grp->obj_oh, NULL /*event*/))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "can't open root group: %d", ret) /* Handle pool_info and container_info DSMINC */ } /* end else */ /* Open global metadata object */ - daos_obj_id_generate(&oid, DAOS_OC_REPLICA_RW); - if(0 != (ret = daos_obj_open(file->glob_md_oh, oid, 0, DAOS_OO_RW, &file->glob_md_oh, NULL /*event*/))) + if(0 != (ret = daos_obj_open(file->glob_md_oh, gmd_oid, 0, DAOS_OO_RW, &file->glob_md_oh, NULL /*event*/))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "can't open global metadata object: %d", ret) /* Finish setting up file struct */ @@ -937,6 +1004,22 @@ H5VL_daosm_file_open(const char *name, unsigned flags, hid_t fapl_id, if(FAIL == H5FD_mpi_comm_info_dup(fa->comm, fa->info, &file->comm, &file->info)) HGOTO_ERROR(H5E_INTERNAL, H5E_CANTCOPY, NULL, "Communicator/Info duplicate failed"); + /* Determine if we want to acquire a transaction for the opened file */ + if(H5P_get(plist, H5VL_ACQUIRE_TR_ID, &trans_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get property value for trans id"); + + if(FAIL != trans_id) { + /* Transaction requested */ + H5TR_t *tr = NULL; + + /* get the TR object */ + if(NULL == (tr = (H5TR_t *)H5I_object_verify(trans_id, H5I_TR))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "not a TRANSACTION ID"); + + tr->epoch = epoch; + tr->file = file; + } /* end else */ + ret_value = (void *)file; done: @@ -946,12 +1029,11 @@ done: /* If the operation is synchronous and it failed at the server, or it failed * locally, then cleanup and return fail */ if(NULL == ret_value) { - /* Bcast gh_sizes as '0' if necessary - this will trigger failures in - * the other processes so we do not need to do the second bcast. */ + /* Bcast bcast_buf_64 as '0' if necessary - this will trigger failures + * in the other processes so we do not need to do the second bcast. */ if(must_bcast) { - gh_sizes[0] = 0; - gh_sizes[1] = 0; - if(MPI_SUCCESS != MPI_Bcast(gh_sizes, 2, MPI_UINT64_T, 0, fa->comm)) + HDmemset(bcast_buf_64, 0, sizeof(bcast_buf_64)); + if(MPI_SUCCESS != MPI_Bcast(bcast_buf_64, sizeof(bcast_buf_64)/sizeof(bcast_buf_64[0]), MPI_UINT64_T, 0, fa->comm)) HDONE_ERROR(H5E_FILE, H5E_MPI, NULL, "can't bcast global handle sizes") } /* end if */ -- cgit v0.12