From 305ac8886566968ae58ba771e0ae2ba6434ac9f0 Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Fri, 14 Jul 2023 22:15:11 +0200 Subject: ROS3: (feature) Temporary security credentials (#3030) - Implemented support for AWS temporary security credentials. For this kind of credentials also a session/security token should be included in the request by adding the x-amz-security-token header. Co-authored-by: Larry Knox Co-authored-by: Jordan Henderson Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- release_docs/RELEASE.txt | 9 + src/H5FDros3.c | 283 ++++++++++++++++++++++++++++++- src/H5FDros3.h | 109 +++++++++++- src/H5FDs3comms.c | 109 +++++++++--- src/H5FDs3comms.h | 3 +- test/s3comms.c | 42 ++--- tools/lib/h5tools.c | 6 +- tools/lib/h5tools_utils.c | 85 ++++++---- tools/lib/h5tools_utils.h | 12 +- tools/libtest/h5tools_test_utils.c | 338 +++++++++++++++++++------------------ tools/src/h5dump/h5dump.c | 15 +- tools/src/h5ls/h5ls.c | 15 +- tools/src/h5stat/h5stat.c | 15 +- 13 files changed, 769 insertions(+), 272 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 35acc34..1c12071 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -169,6 +169,15 @@ New Features Library: -------- + - Implemented support for temporary security credentials for the Read-Only + S3 (ROS3) file driver. + + When using temporary security credentials, one also needs to specify a + session/security token next to the access key id and secret access key. + This token can be specified by the new API function H5Pset_fapl_ros3_token(). + The API function H5Pget_fapl_ros3_token() can be used to retrieve + the currently set token. + - Added a Subfiling VFD configuration file prefix environment variable The Subfiling VFD now checks for values set in a new environment diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 3628185e..be35abc 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -47,6 +47,10 @@ */ static hid_t H5FD_ROS3_g = 0; +/* Session/security token property name + */ +#define ROS3_TOKEN_PROP_NAME "ros3_token_prop" + #if ROS3_STATS /* arbitrarily large value, such that any reasonable size read will be "less" @@ -225,6 +229,11 @@ static herr_t H5FD__ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing static herr_t H5FD__ros3_validate_config(const H5FD_ros3_fapl_t *fa); +static herr_t H5FD__ros3_str_token_copy(const char *name, size_t size, void *_value); +static int H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t size); +static herr_t H5FD__ros3_str_token_close(const char *name, size_t size, void *_value); +static herr_t H5FD__ros3_str_token_delete(hid_t prop_id, const char *name, size_t size, void *_value); + static const H5FD_class_t H5FD_ros3_g = { H5FD_CLASS_VERSION, /* struct version */ H5FD_ROS3_VALUE, /* value */ @@ -566,6 +575,257 @@ H5FD__ros3_fapl_free(void *_fa) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD__ros3_fapl_free() */ +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_ros3_token + * + * Purpose: Returns session/security token of the ros3 file access + * property list though the function arguments. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) +{ + H5P_genplist_t *plist = NULL; + char *token_src; + htri_t token_exists; + size_t tokenlen = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "izx", fapl_id, size, token_dst); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pget_fapl_ros3_token() called.\n"); +#endif + + if (size == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size cannot be zero.") + if (token_dst == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "token_dst is NULL") + + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5FD_ROS3 != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") + } + + /* Copy the token data out */ + tokenlen = HDstrlen(token_src); + if (size <= tokenlen) { + tokenlen = size - 1; + } + H5MM_memcpy(token_dst, token_src, sizeof(char) * tokenlen); + token_dst[tokenlen] = '\0'; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_fapl_ros3_token() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_copy() + * + * Purpose: Create a copy of the token string. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value) +{ + char **value = (char **)_value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD__ros3_str_token_copy() called.\n"); +#endif + + if (*value) + if (NULL == (*value = HDstrdup(*value))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy string property token") + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD__ros3_str_token_copy() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_cmp() + * + * Purpose: Compares two token strings with each other. + * + * Return: + * - Equivalent: 0 + * - Not Equivalent: non-zero value + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static int +H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t H5_ATTR_UNUSED size) +{ + char *const *value1 = (char *const *)_value1; + char *const *value2 = (char *const *)_value2; + int ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + if (*value1) { + if (*value2) + ret_value = HDstrcmp(*value1, *value2); + else + ret_value = 1; + } + else { + if (*value2) + ret_value = -1; + else + ret_value = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_cmp */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_close() + * + * Purpose: Closes/frees the memory associated to the token string. + * Currently, it is an empty implementation since there no + * additional treatment needed for this property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value) +{ + char **value = (char **)_value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + if (*value) + free(*value); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_close */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_delete() + * + * Purpose: Deletes the property token from the property list and frees + * the memory associated to the token string. + * Currently, it is an empty implementation since there no + * additional treatment needed for this property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_delete(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *_value) +{ + char **value = (char **)_value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + if (*value) + free(*value); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_delete */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_ros3_token() + * + * Purpose: Modify the file access property list to use the H5FD_ROS3 + * driver defined in this source file by adding or + * modifying the session/security token property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) +{ + H5P_genplist_t *plist = NULL; + char *token_src; + htri_t token_exists; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*s", fapl_id, token); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pset_fapl_ros3_token() called.\n"); +#endif + + if (fapl_id == H5P_DEFAULT) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list") + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5FD_ROS3 != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if (HDstrlen(token) > H5FD_ROS3_MAX_SECRET_TOK_LEN) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, + "specified token exceeds the internally specified maximum string length") + + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") + + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") + + memcpy(token_src, token, HDstrlen(token) + 1); + } + else { + token_src = (char *)malloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1)); + if (token_src == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for token_src variable."); + memcpy(token_src, token, HDstrlen(token) + 1); + if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char *), &token_src, NULL, NULL, NULL, NULL, + H5FD__ros3_str_token_delete, H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, + H5FD__ros3_str_token_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist") + } + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_ros3_token() */ + #if ROS3_STATS /*---------------------------------------------------------------------------- * @@ -654,6 +914,9 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) unsigned char signing_key[SHA256_DIGEST_LENGTH]; s3r_t *handle = NULL; H5FD_ros3_fapl_t fa; + H5P_genplist_t *plist = NULL; + htri_t token_exists; + char *token; H5FD_t *ret_value = NULL; FUNC_ENTER_PACKAGE @@ -681,6 +944,16 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)") + /* Session/security token */ + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, NULL, "not a file access property list") + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "failed to check if property token exists in plist") + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get token value") + } + /* open file; procedure depends on whether or not the fapl instructs to * authenticate requests or not. */ @@ -697,11 +970,15 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) (const char *)fa.aws_region, (const char *)iso8601now)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key") - handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, - (const unsigned char *)signing_key); + if (token_exists) + handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, + (const unsigned char *)signing_key, (const char *)token); + else + handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, + (const unsigned char *)signing_key, ""); } else - handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL, NULL); if (handle == NULL) /* If we want to check CURL's say on the matter in a controlled diff --git a/src/H5FDros3.h b/src/H5FDros3.h index f84b1a4..0f2f315 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -70,12 +70,63 @@ * ****************************************************************************/ +/** + * \def H5FD_CURR_ROS3_FAPL_T_VERSION + * The version number of the H5FD_ros3_fapl_t configuration + * structure for the $H5FD_ROS3 driver. + */ #define H5FD_CURR_ROS3_FAPL_T_VERSION 1 -#define H5FD_ROS3_MAX_REGION_LEN 32 -#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +/** + * \def H5FD_ROS3_MAX_REGION_LEN + * Maximum string length for specifying the region of the S3 bucket. + */ +#define H5FD_ROS3_MAX_REGION_LEN 32 +/** + * \def H5FD_ROS3_MAX_SECRET_ID_LEN + * Maximum string length for specifying the security ID. + */ +#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +/** + * \def H5FD_ROS3_MAX_SECRET_KEY_LEN + * Maximum string length for specifying the security key. + */ #define H5FD_ROS3_MAX_SECRET_KEY_LEN 128 +/** + * \def H5FD_ROS3_MAX_SECRET_TOK_LEN + * Maximum string length for specifying the session/security token. + */ +#define H5FD_ROS3_MAX_SECRET_TOK_LEN 1024 +/** + *\struct H5FD_ros3_fapl_t + * \brief Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3(). + * + * \details H5FD_ros_fapl_t is a public structure that is used to pass + * configuration data to the #H5FD_ROS3 driver via a File Access + * Property List. A pointer to an instance of this structure is + * a parameter to H5Pset_fapl_ros3() and H5Pget_fapl_ros3(). + * + * \var int32_t H5FD_ros3_fapl_t::version + * Version number of the H5FD_ros3_fapl_t structure. Any instance passed + * to H5Pset_fapl_ros3() / H5Pget_fapl_ros3() must have a recognized version + * number or an error will be raised. Currently, this field should be set + * to #H5FD_CURR_ROS3_FAPL_T_VERSION. + * + * \var hbool_t H5FD_ros3_fapl_t::authenticate + * A boolean which specifies if security credentials should be used for + * accessing a S3 bucket. + * + * \var char H5FD_ros3_fapl_t::aws_region[H5FD_ROS3_MAX_REGION_LEN + 1] + * A string which specifies the AWS region of the S3 bucket. + * + * \var char H5FD_ros3_fapl_t::secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1] + * A string which specifies the security ID. + * + * \var char H5FD_ros3_fapl_t::secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1] + * A string which specifies the security key. + * + */ typedef struct H5FD_ros3_fapl_t { int32_t version; hbool_t authenticate; @@ -88,22 +139,72 @@ typedef struct H5FD_ros3_fapl_t { extern "C" { #endif +/** + * \brief Internal routine to initialize #H5FD_ROS3 driver. Not meant to be + * called directly by an HDF5 application. + */ H5_DLL hid_t H5FD_ros3_init(void); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Queries a File Access Property List for #H5FD_ROS3 file driver properties. + * + * \fapl_id + * \param[out] fa_out Pointer to #H5FD_ROS3 driver configuration structure. + * \returns \herr_t */ H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver. + * + * \fapl_id + * \param[in] fa Pointer to #H5FD_ROS3 driver configuration structure. + * \returns \herr_t */ H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa); +/** + * \ingroup FAPL + * + * \brief Queries a File Access Property List for #H5FD_ROS3 file driver session/security + * token. + * + * \fapl_id + * \param[in] size Size of the provided char array for storing the session/security token. + * \param[out] token Session/security token. + * \returns \herr_t + * + * \since 1.14.2 + */ +H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token); + +/** + * \ingroup FAPL + * + * \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver + * by adding the specified session/security token. + * + * \fapl_id + * \param[in] token Session/security token. + * \returns \herr_t + * + * \details H5Pset_fapl_ros3_token() modifies an existing File Access Property List which + * is used by #H5FD_ROS3 driver by adding or updating the session/security token + * of the property list. Be aware, to set the token first you need to create + * a proper File Access Property List using H5Pset_fapl_ros() and use this list + * as input argument of the function H5Pset_fapl_ros3_token(). + * + * Note, the session token is only needed when you want to access a S3 bucket + * using temporary security credentials. + * + * \since 1.14.2 + */ +H5_DLL herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token); + #ifdef __cplusplus } #endif diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 74a4bbc..10b3ff2 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -39,6 +39,7 @@ #include "H5Eprivate.h" /* error handling */ #include "H5MMprivate.h" /* memory management */ #include "H5FDs3comms.h" /* S3 Communications */ +#include "H5FDros3.h" /* ros3 file driver */ /****************/ /* Local Macros */ @@ -775,6 +776,7 @@ H5FD_s3comms_s3r_close(s3r_t *handle) H5MM_xfree(handle->secret_id); H5MM_xfree(handle->region); H5MM_xfree(handle->signing_key); + H5MM_xfree(handle->token); assert(handle->httpverb != NULL); H5MM_xfree(handle->httpverb); @@ -1001,7 +1003,8 @@ done: *---------------------------------------------------------------------------- */ s3r_t * -H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key) +H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key, + const char *token) { size_t tmplen = 0; CURL *curlh = NULL; @@ -1035,13 +1038,15 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const handle->region = NULL; handle->secret_id = NULL; handle->signing_key = NULL; + handle->token = NULL; handle->httpverb = NULL; /************************************* * RECORD AUTHENTICATION INFORMATION * *************************************/ - if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL)) { + if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) || + (token != NULL)) { /* if one exists, all three must exist */ if (region == NULL || region[0] == '\0') @@ -1050,6 +1055,8 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "secret id cannot be null."); if (signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "signing key cannot be null."); + if (token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "token cannot be null."); /* copy strings */ tmplen = HDstrlen(region) + 1; @@ -1069,6 +1076,12 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy."); H5MM_memcpy(handle->signing_key, signing_key, tmplen); + + tmplen = HDstrlen(token) + 1; + handle->token = (char *)H5MM_malloc(sizeof(char) * tmplen); + if (handle->token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle token copy."); + H5MM_memcpy(handle->token, token, tmplen); } /* if authentication information provided */ /************************ @@ -1130,6 +1143,7 @@ done: H5MM_xfree(handle->region); H5MM_xfree(handle->secret_id); H5MM_xfree(handle->signing_key); + H5MM_xfree(handle->token); if (handle->httpverb != NULL) H5MM_xfree(handle->httpverb); H5MM_xfree(handle); @@ -1190,8 +1204,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) hrb_t *request = NULL; int ret = 0; /* working variable to check */ /* return value of HDsnprintf */ - struct s3r_datastruct *sds = NULL; - herr_t ret_value = SUCCEED; + char *authorization = NULL; + char *buffer1 = NULL; + char *signed_headers = NULL; + struct s3r_datastruct *sds = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -1284,23 +1301,31 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) else { /* authenticate request */ - char authorization[512 + 1]; - /* 512 := approximate max length... - * 67 - * + 8 - * + 64 - * + 128 - * + 20 - * + 128 + authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1); + if (authorization == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for authorization variable."); + /* 2048 := approximate max length... + * 67 + * + 8 + * + 64 + * + 128 + * + 20 + * + 128 + * + 1024 */ - char buffer1[512 + 1]; /* -> Canonical Request -> Signature */ char buffer2[256 + 1]; /* -> String To Sign -> Credential */ char iso8601now[ISO8601_SIZE]; - char signed_headers[48 + 1]; + buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + + 1); /* -> Canonical Request -> Signature */ + if (buffer1 == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for buffer1 variable."); + signed_headers = (char *)H5MM_malloc(48 + H5FD_ROS3_MAX_SECRET_KEY_LEN + 1); + if (signed_headers == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for signed_headers variable."); /* should be large enough for nominal listing: - * "host;range;x-amz-content-sha256;x-amz-date" - * + '\0', with "range;" possibly absent + * "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token" + * + '\0', with "range;" and/or "x-amz-security-token" possibly absent */ /* zero start of strings */ @@ -1318,6 +1343,8 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null secret_id."); if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null signing_key."); + if (handle->token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null token."); if (handle->httpverb == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null httpverb."); if (handle->purl->host == NULL) @@ -1349,6 +1376,15 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list."); assert(headers->magic == S3COMMS_HRB_NODE_MAGIC); + if (HDstrlen((const char *)handle->token) > 0) { + if (FAIL == + H5FD_s3comms_hrb_node_set(&headers, "x-amz-security-token", (const char *)handle->token)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header") + if (headers == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list."); + assert(headers->magic == S3COMMS_HRB_NODE_MAGIC); + } + if (rangebytesstr != NULL) { if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "Range", rangebytesstr)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set range header") @@ -1368,8 +1404,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) /**** COMPUTE AUTHORIZATION ****/ /* buffer1 -> canonical request */ - if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512, signed_headers, 48, request)) + if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + signed_headers, 48 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + request)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad canonical request"); + } /* buffer2->string-to-sign */ if (FAIL == H5FD_s3comms_tostringtosign(buffer2, buffer1, iso8601now, handle->region)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad string-to-sign"); @@ -1383,9 +1422,10 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string"); - ret = HDsnprintf(authorization, 512, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", - buffer2, signed_headers, buffer1); - if (ret <= 0 || ret >= 512) + ret = HDsnprintf(authorization, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2, + signed_headers, buffer1); + if (ret <= 0 || ret >= 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string"); /* append authorization header to http request buffer */ @@ -1440,8 +1480,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) if (p_status != CURLE_OK) { if (CURLE_OK != curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem getting response code") - fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %d\n", p_status, httpcode); + fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %ld\n", p_status, httpcode); fprintf(stdout, "%s\n", curl_easy_strerror(p_status)); + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "problem while performing request."); } if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL)) @@ -1474,6 +1515,18 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) done: /* clean any malloc'd resources */ + if (authorization != NULL) { + H5MM_xfree(authorization); + authorization = NULL; + } + if (buffer1 != NULL) { + H5MM_xfree(buffer1); + buffer1 = NULL; + } + if (signed_headers != NULL) { + H5MM_xfree(signed_headers); + signed_headers = NULL; + } if (curlheaders != NULL) { curl_slist_free_all(curlheaders); curlheaders = NULL; @@ -1592,8 +1645,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c size_t sh_size = (size_t)_sh_size; size_t cr_len = 0; /* working length of canonical request str */ size_t sh_len = 0; /* working length of signed headers str */ - char tmpstr[256 + 1]; - tmpstr[256] = 0; /* terminating NULL */ + char tmpstr[1024]; /* "query params" refers to the optional element in the URL, e.g. * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J @@ -1626,6 +1678,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c (size_t)3); /* three newline chars */ if (cr_len >= cr_size) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request"); + /* TODO: compiler warning */ ret = HDsnprintf(canonical_request_dest, (cr_size - 1), "%s\n%s\n%s\n", http_request->verb, http_request->resource, query_params); @@ -1638,8 +1691,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c assert(node->magic == S3COMMS_HRB_NODE_MAGIC); - ret = HDsnprintf(tmpstr, 256, "%s:%s\n", node->lowername, node->value); - if (ret < 0 || ret >= 256) + ret = HDsnprintf(tmpstr, 1024, "%s:%s\n", node->lowername, node->value); + if (ret < 0 || ret >= 1024) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s", node->lowername, node->value); cr_len += HDstrlen(tmpstr); @@ -1647,8 +1700,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request"); HDstrcat(canonical_request_dest, tmpstr); - ret = HDsnprintf(tmpstr, 256, "%s;", node->lowername); - if (ret < 0 || ret >= 256) + ret = HDsnprintf(tmpstr, 1024, "%s;", node->lowername); + if (ret < 0 || ret >= 1024) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to append semicolon to lowername %s", node->lowername); sh_len += HDstrlen(tmpstr); diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h index 1b21ad7..47857f6 100644 --- a/src/H5FDs3comms.h +++ b/src/H5FDs3comms.h @@ -478,6 +478,7 @@ typedef struct { char *region; char *secret_id; unsigned char *signing_key; + char *token; } s3r_t; #define S3COMMS_S3R_MAGIC 0x44d8d79 @@ -509,7 +510,7 @@ H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle); H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle); H5_DLL s3r_t *H5FD_s3comms_s3r_open(const char url[], const char region[], const char id[], - const unsigned char signing_key[]); + const unsigned char signing_key[], const char token[]); H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest); diff --git a/test/s3comms.c b/test/s3comms.c index 07dcce9..ea32769 100644 --- a/test/s3comms.c +++ b/test/s3comms.c @@ -348,6 +348,7 @@ static int s3_test_credentials_loaded = 0; static char s3_test_aws_region[16] = ""; static char s3_test_aws_access_key_id[64] = ""; static char s3_test_aws_secret_access_key[128] = ""; +static char s3_test_aws_security_token[1024] = ""; static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; static hbool_t s3_test_bucket_defined = FALSE; @@ -1722,7 +1723,7 @@ test_s3r_get_filesize(void) JSVERIFY(0, H5FD_s3comms_s3r_get_filesize(NULL), "filesize of the null handle should be 0") - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL) JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) @@ -1830,14 +1831,14 @@ test_s3r_open(void) /* attempt anonymously */ - handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); /* attempt with authentication */ - handle = - H5FD_s3comms_s3r_open(url_missing, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_missing, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /************************* @@ -1846,7 +1847,7 @@ test_s3r_open(void) #if S3_TEST_RUN_TIMEOUT printf("Opening on inactive port may hang for a minute; waiting for timeout\n"); - handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); #endif @@ -1856,20 +1857,21 @@ test_s3r_open(void) /* anonymous access on restricted file */ - handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); /* passed in a bad ID */ - handle = H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID", - (const unsigned char *)signing_key); + handle = + H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID", + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /* using an invalid signing key */ - handle = - H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)EMPTY_SHA256); + handle = H5FD_s3comms_s3r_open( + url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /******************************* @@ -1878,7 +1880,7 @@ test_s3r_open(void) /* anonymous */ - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL); JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), "did not get expected filesize") JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1886,9 +1888,9 @@ test_s3r_open(void) /* using authentication on anonymously-accessible file? */ - handle = - H5FD_s3comms_s3r_open(url_raven, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_raven, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle == NULL); JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1896,9 +1898,9 @@ test_s3r_open(void) /* authenticating */ - handle = - H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle == NULL); JSVERIFY(5458199, H5FD_s3comms_s3r_get_filesize(handle), NULL) JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1984,7 +1986,7 @@ test_s3r_read(void) /* open file */ - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL) JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 5beffa4..76f3839 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -565,8 +565,12 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info) #ifdef H5_HAVE_ROS3_VFD if (!vfd_info->info) H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid"); - if (H5Pset_fapl_ros3(fapl_id, (const H5FD_ros3_fapl_t *)vfd_info->info) < 0) + if (H5Pset_fapl_ros3(fapl_id, &((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->fa) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed"); + + if (H5Pset_fapl_ros3_token(fapl_id, ((const H5FD_ros3_fapl_ext_t *)vfd_info->info)->token) < + 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3_token() failed"); #else H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled"); #endif diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index e9b6dfc..fc9a646 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -1025,9 +1025,9 @@ done: *---------------------------------------------------------------------------- */ herr_t -h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t *fapl_config_out) +h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_ext_t *fapl_config_out) { - const char *ccred[3]; + const char *ccred[4]; unsigned nelems = 0; char *s3cred_src = NULL; char **s3cred = NULL; @@ -1038,12 +1038,18 @@ h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, H5FD_ros3_fapl_t H5TOOLS_GOTO_ERROR(FAIL, "failed to parse S3 VFD info tuple"); /* Sanity-check tuple count */ - if (nelems != 3) + if (nelems != 3 && nelems != 4) H5TOOLS_GOTO_ERROR(FAIL, "invalid S3 VFD credentials"); ccred[0] = (const char *)s3cred[0]; ccred[1] = (const char *)s3cred[1]; ccred[2] = (const char *)s3cred[2]; + if (nelems == 3) { + ccred[3] = ""; + } + else { + ccred[3] = (const char *)s3cred[3]; + } if (0 == h5tools_populate_ros3_fapl(fapl_config_out, ccred)) H5TOOLS_GOTO_ERROR(FAIL, "failed to populate S3 VFD FAPL config"); @@ -1090,34 +1096,35 @@ done: * * NULL fapl pointer: (NULL, {...} ) * * Warning: In all cases below, fapl will be set as "default" * before error occurs. - * * NULL value strings: (&fa, {NULL?, NULL? NULL?, ...}) + * * NULL value strings: (&fa, {NULL?, NULL? NULL?, NULL?, ...}) * * Incomplete fapl info: - * * empty region, non-empty id, key either way - * * (&fa, {"", "...", "?"}) - * * empty id, non-empty region, key either way - * * (&fa, {"...", "", "?"}) - * * "non-empty key and either id or region empty - * * (&fa, {"", "", "...") - * * (&fa, {"", "...", "...") - * * (&fa, {"...", "", "...") + * * empty region, non-empty id, key either way, token either way + * * (&fa, token, {"", "...", "?", "?"}) + * * empty id, non-empty region, key either way, token either way + * * (&fa, token, {"...", "", "?", "?"}) + * * "non-empty key, token either way and either id or region empty + * * (&fa, token, {"", "", "...", "?") + * * (&fa, token, {"", "...", "...", "?") + * * (&fa, token, {"...", "", "...", "?") * * Any string would overflow allowed space in fapl definition. * or * 1 (success) * * Sets components in fapl_t pointer, copying strings as appropriate. * * "Default" fapl (valid version, authenticate->False, empty strings) * * `values` pointer is NULL - * * (&fa, NULL) - * * first three strings in `values` are empty ("") - * * (&fa, {"", "", "", ...} + * * (&fa, token, NULL) + * * first four strings in `values` are empty ("") + * * (&fa, token, {"", "", "", "", ...}) * * Authenticating fapl - * * region, id, and optional key provided - * * (&fa, {"...", "...", ""}) - * * (&fa, {"...", "...", "..."}) + * * region, id, optional key and option session token provided + * * (&fa, token, {"...", "...", "", ""}) + * * (&fa, token, {"...", "...", "...", ""}) + * * (&fa, token, {"...", "...", "...", "..."}) * *---------------------------------------------------------------------------- */ int -h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) +h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values) { int show_progress = 0; /* set to 1 for debugging */ int ret_value = 1; /* 1 for success, 0 for failure */ @@ -1138,11 +1145,12 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) if (show_progress) { printf(" preset fapl with default values\n"); } - fa->version = H5FD_CURR_ROS3_FAPL_T_VERSION; - fa->authenticate = FALSE; - *(fa->aws_region) = '\0'; - *(fa->secret_id) = '\0'; - *(fa->secret_key) = '\0'; + fa->fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION; + fa->fa.authenticate = FALSE; + *(fa->fa.aws_region) = '\0'; + *(fa->fa.secret_id) = '\0'; + *(fa->fa.secret_key) = '\0'; + *(fa->token) = '\0'; /* sanity-check supplied values */ @@ -1168,6 +1176,13 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) ret_value = 0; goto done; } + if (values[3] == NULL) { + if (show_progress) { + printf(" ERROR: token value cannot be NULL\n"); + } + ret_value = 0; + goto done; + } /* if region and ID are supplied (key optional), write to fapl... * fail if value would overflow @@ -1180,7 +1195,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) ret_value = 0; goto done; } - memcpy(fa->aws_region, values[0], (HDstrlen(values[0]) + 1)); + memcpy(fa->fa.aws_region, values[0], (HDstrlen(values[0]) + 1)); if (show_progress) { printf(" aws_region set\n"); } @@ -1192,7 +1207,7 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) ret_value = 0; goto done; } - memcpy(fa->secret_id, values[1], (HDstrlen(values[1]) + 1)); + memcpy(fa->fa.secret_id, values[1], (HDstrlen(values[1]) + 1)); if (show_progress) { printf(" secret_id set\n"); } @@ -1204,17 +1219,29 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) ret_value = 0; goto done; } - memcpy(fa->secret_key, values[2], (HDstrlen(values[2]) + 1)); + memcpy(fa->fa.secret_key, values[2], (HDstrlen(values[2]) + 1)); if (show_progress) { printf(" secret_key set\n"); } - fa->authenticate = TRUE; + if (HDstrlen(values[3]) > H5FD_ROS3_MAX_SECRET_TOK_LEN) { + if (show_progress) { + printf(" ERROR: token value too long\n"); + } + ret_value = 0; + goto done; + } + memcpy(fa->token, values[3], (HDstrlen(values[3]) + 1)); + if (show_progress) { + printf(" token set\n"); + } + + fa->fa.authenticate = TRUE; if (show_progress) { printf(" set to authenticate\n"); } } - else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0') { + else if (*values[0] != '\0' || *values[1] != '\0' || *values[2] != '\0' || *values[3] != '\0') { if (show_progress) { printf(" ERROR: invalid assortment of empty/non-empty values\n"); } diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index c721ab4..2efa53f 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -61,6 +61,14 @@ typedef struct find_objs_t { table_t *dset_table; } find_objs_t; +#ifdef H5_HAVE_ROS3_VFD +/*extended configuration struct for holding the configuration data to the #H5FD_ROS3 driver */ +typedef struct H5FD_ros3_fapl_ext_t { + H5FD_ros3_fapl_t fa; /* ROS3 configuration struct*/ + char token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; /* Session/security token*/ +} H5FD_ros3_fapl_ext_t; +#endif /* H5_HAVE_ROS3_VFD */ + H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for outputting */ /* Definitions of useful routines */ @@ -126,8 +134,8 @@ H5TOOLS_DLL void h5tools_setstatus(int d_status); H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void); #ifdef H5_HAVE_ROS3_VFD H5TOOLS_DLL herr_t h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, - H5FD_ros3_fapl_t *fapl_config_out); -H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values); + H5FD_ros3_fapl_ext_t *fapl_config_out); +H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_ext_t *fa, const char **values); #endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS H5TOOLS_DLL herr_t h5tools_parse_hdfs_fapl_tuple(const char *tuple_str, int delim, diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c index d1da145..87c2ce9 100644 --- a/tools/libtest/h5tools_test_utils.c +++ b/tools/libtest/h5tools_test_utils.c @@ -22,10 +22,6 @@ #define UTIL_TEST_DEBUG 0 -#ifndef __js_test__ - -#define __js_test__ 1L - /***************************************************************************** * * FILE-LOCAL TESTING MACROS @@ -59,7 +55,6 @@ * * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order * FAIL_IF() - check condition - * FAIL_UNLESS() - check _not_ condition * JSVERIFY() - long-int equality check; prints reason/comparison * JSVERIFY_NOT() - long-int inequality check; prints * JSVERIFY_STR() - string equality check; prints @@ -125,31 +120,6 @@ H5_GCC_CLANG_DIAG_OFF("format") /*---------------------------------------------------------------------------- * - * Macro: FAIL_UNLESS() - * - * Purpose: - * - * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", - * e.g., "a != b". - * - * Opposite of FAIL_IF; fails if the given condition is _not_ true. - * - * `FAIL_IF( 5 != my_op() )` - * is equivalent to - * `FAIL_UNLESS( 5 == my_op() )` - * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. - * (see JSVERIFY) - * - *---------------------------------------------------------------------------- - */ -#define FAIL_UNLESS(condition) \ - if (!(condition)) { \ - JSFAILED_AT() \ - goto error; \ - } - -/*---------------------------------------------------------------------------- - * * Macro: JSERR_LONG() * * Purpose: @@ -314,8 +284,6 @@ H5_GCC_CLANG_DIAG_OFF("format") #endif /* ifdef/else JSVERIFY_EXP_ACT */ -#endif /* __js_test__ */ - /* if > 0, be very verbose when performing tests */ #define H5TOOLS_UTILS_TEST_DEBUG 0 @@ -508,11 +476,11 @@ test_parse_tuple(void) success = parse_tuple(tc.in_str, tc.sep, &cpy, &count, &parsed); JSVERIFY(tc.exp_ret, success, "function returned incorrect value") - JSVERIFY(tc.exp_nelems, count, NULL) + JSVERIFY(tc.exp_nelems, count, (char *)NULL) if (success == SUCCEED) { FAIL_IF(parsed == NULL) for (elem_i = 0; elem_i < count; elem_i++) { - JSVERIFY_STR(tc.exp_elems[elem_i], parsed[elem_i], NULL) + JSVERIFY_STR(tc.exp_elems[elem_i], parsed[elem_i], (char *)NULL) } /* TEARDOWN */ assert(parsed != NULL); @@ -605,104 +573,109 @@ test_populate_ros3_fa(void) /* NULL values pointer yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, "x"}; if (show_progress) { printf("NULL values pointer\n"); } JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, NULL), "NULL values pointer yields \"default\" fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* all-empty values * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; - const char *values[] = {"", "", ""}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, TRUE, "u", "v", "w"}, "x"}; + const char *values[] = {"", "", "", ""}; if (show_progress) { printf("all empty values\n"); } JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "empty values yields \"default\" fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* successfully set fapl with values * excess value is ignored */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", "z", "a"}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "z", "a", "b"}; if (show_progress) { printf("successful full set\n"); } JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "four values") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(TRUE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("y", fa.secret_id, NULL) - JSVERIFY_STR("z", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(TRUE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("z", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("a", fa.token, (char *)NULL) } /* NULL region * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {NULL, "y", "z", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {NULL, "y", "z", ""}; if (show_progress) { printf("NULL region\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty region * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"", "y", "z", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"", "y", "z", ""}; if (show_progress) { printf("empty region; non-empty id, key\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* region overflow * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"somewhere over the rainbow not too high " - "there is another rainbow bounding some darkened sky", - "y", "z"}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"somewhere over the rainbow not too high " + "there is another rainbow bounding some darkened sky", + "y", "z", ""}; if (show_progress) { printf("region overflow\n"); @@ -711,68 +684,71 @@ test_populate_ros3_fa(void) assert(HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN); JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* NULL id * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", NULL, "z", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", NULL, "z", ""}; if (show_progress) { printf("NULL id\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty id (non-empty region, key) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "", "z", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "", "z", ""}; if (show_progress) { printf("empty id; non-empty region and key\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* id overflow * partial set: region */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "Why is it necessary to solve the problem? " - "What benefits will you receive by solving the problem? " - "What is the unknown? " - "What is it you don't yet understand? " - "What is the information you have? " - "What isn't the problem? " - "Is the information insufficient, redundant, or contradictory? " - "Should you draw a diagram or figure of the problem? " - "What are the boundaries of the problem? " - "Can you separate the various parts of the problem?", - "z"}; + "What benefits will you receive by solving the problem? " + "What is the unknown? " + "What is it you don't yet understand? " + "What is the information you have? " + "What isn't the problem? " + "Is the information insufficient, redundant, or contradictory? " + "Should you draw a diagram or figure of the problem? " + "What are the boundaries of the problem? " + "Can you separate the various parts of the problem?", + "z", ""}; if (show_progress) { printf("id overflow\n"); @@ -781,105 +757,131 @@ test_populate_ros3_fa(void) assert(HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN); JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* NULL key * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", NULL, NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", NULL, ""}; if (show_progress) { printf("NULL key\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) + } + + /* NULL token + * yields default fapl + */ + { + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "z", NULL}; + + if (show_progress) { + printf("NULL key\n"); + } + + JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill token") + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty key (non-empty region, id) * yields authenticating fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", "", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "", ""}; if (show_progress) { printf("empty key; non-empty region and id\n"); } JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(TRUE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("y", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(TRUE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty key, region (non-empty id) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"", "y", "", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"", "y", "", ""}; if (show_progress) { printf("empty key and region; non-empty id\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* empty key, id (non-empty region) * yields default fapl */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "", "", NULL}; + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "", "", ""}; if (show_progress) { printf("empty key and id; non-empty region\n"); } JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("", fa.aws_region, NULL) - JSVERIFY_STR("", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* key overflow * partial set: region, id */ { - H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; - const char *values[] = {"x", "y", + H5FD_ros3_fapl_ext_t fa = {{bad_version, FALSE, "a", "b", "c"}, "d"}; + const char *values[] = {"x", "y", "Why is it necessary to solve the problem? " - "What benefits will you receive by solving the problem? " - "What is the unknown? " - "What is it you don't yet understand? " - "What is the information you have? " - "What isn't the problem? " - "Is the information insufficient, redundant, or contradictory? " - "Should you draw a diagram or figure of the problem? " - "What are the boundaries of the problem? " - "Can you separate the various parts of the problem?"}; + "What benefits will you receive by solving the problem? " + "What is the unknown? " + "What is it you don't yet understand? " + "What is the information you have? " + "What isn't the problem? " + "Is the information insufficient, redundant, or contradictory? " + "Should you draw a diagram or figure of the problem? " + "What are the boundaries of the problem? " + "Can you separate the various parts of the problem?", + ""}; if (show_progress) { printf("key overflow\n"); @@ -888,22 +890,23 @@ test_populate_ros3_fa(void) assert(HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN); JSVERIFY(0, h5tools_populate_ros3_fapl(&fa, values), "could not fill fapl") - JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL) - JSVERIFY(FALSE, fa.authenticate, NULL) - JSVERIFY_STR("x", fa.aws_region, NULL) - JSVERIFY_STR("y", fa.secret_id, NULL) - JSVERIFY_STR("", fa.secret_key, NULL) + JSVERIFY(H5FD_CURR_ROS3_FAPL_T_VERSION, fa.fa.version, (char *)NULL) + JSVERIFY(FALSE, fa.fa.authenticate, (char *)NULL) + JSVERIFY_STR("x", fa.fa.aws_region, (char *)NULL) + JSVERIFY_STR("y", fa.fa.secret_id, (char *)NULL) + JSVERIFY_STR("", fa.fa.secret_key, (char *)NULL) + JSVERIFY_STR("", fa.token, (char *)NULL) } /* use case */ { - H5FD_ros3_fapl_t fa = {0, 0, "", "", ""}; - const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA", - "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"}; + H5FD_ros3_fapl_ext_t fa = {{0, 0, "", "", ""}, ""}; + const char *values[] = {"us-east-2", "AKIAIMC3D3XLYXLN5COA", + "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+", ""}; JSVERIFY(1, h5tools_populate_ros3_fapl(&fa, values), "unable to set use case") - JSVERIFY(1, fa.version, "version check") - JSVERIFY(1, fa.authenticate, "should authenticate") + JSVERIFY(1, fa.fa.version, "version check") + JSVERIFY(1, fa.fa.authenticate, "should authenticate") } PASSED(); @@ -962,14 +965,7 @@ test_set_configured_fapl(void) hid_t fapl_id = H5I_INVALID_HID; other_fa_t wrong_fa = {0x432, 0xf82, 0x9093}; #ifdef H5_HAVE_ROS3_VFD - H5FD_ros3_fapl_t ros3_anon_fa = {1, FALSE, "", "", ""}; - H5FD_ros3_fapl_t ros3_auth_fa = { - 1, /* fapl version */ - TRUE, /* authenticate */ - "us-east-1", /* aws region */ - "12345677890abcdef", /* simulate access key ID */ - "oiwnerwe9u0234nJw0-aoj+dsf", /* simulate secret key */ - }; + H5FD_ros3_fapl_ext_t ros3_anon_fa = {{1, FALSE, "", "", ""}, ""}; #endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_LIBHDFS H5FD_hdfs_fapl_t hdfs_fa = { @@ -1114,7 +1110,17 @@ test_set_configured_fapl(void) vfd_info.type = VFD_BY_NAME; vfd_info.info = C.conf_fa; vfd_info.u.name = C.vfdname; - result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + + if (C.expected == 1) + result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + else { + H5E_BEGIN_TRY + { + result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info); + } + H5E_END_TRY; + } + if (C.expected == 0) { JSVERIFY(result, H5I_INVALID_HID, C.message) } diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index dd223f3..1458354 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -32,12 +32,15 @@ static hbool_t get_onion_revision_count = FALSE; #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ -static H5FD_ros3_fapl_t ros3_fa_g = { - 1, /* Structure Version */ - FALSE, /* Authenticate? */ - "", /* AWS Region */ - "", /* Access Key ID */ - "", /* Secret Access Key */ +static H5FD_ros3_fapl_ext_t ros3_fa_g = { + { + 1, /* Structure Version */ + FALSE, /* Authenticate? */ + "", /* AWS Region */ + "", /* Access Key ID */ + "", /* Secret Access Key */ + }, + "", /* Session/security token */ }; #endif /* H5_HAVE_ROS3_VFD */ diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index ed81210..b6ce9b1 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -2656,12 +2656,15 @@ main(int argc, char *argv[]) #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ - H5FD_ros3_fapl_t ros3_fa = { - 1, /* Structure Version */ - FALSE, /* Authenticate? */ - "", /* AWS Region */ - "", /* Access Key ID */ - "", /* Secret Access Key */ + H5FD_ros3_fapl_ext_t ros3_fa = { + { + 1, /* Structure Version */ + FALSE, /* Authenticate? */ + "", /* AWS Region */ + "", /* Access Key ID */ + "", /* Secret Access Key */ + }, + "", /* Session/security token */ }; #endif /* H5_HAVE_ROS3_VFD */ diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index f9ca18a..5cfa61d 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -119,12 +119,15 @@ static const char *drivername = NULL; #ifdef H5_HAVE_ROS3_VFD /* Default "anonymous" S3 configuration */ -static H5FD_ros3_fapl_t ros3_fa = { - 1, /* Structure Version */ - FALSE, /* Authenticate? */ - "", /* AWS Region */ - "", /* Access Key ID */ - "", /* Secret Access Key */ +static H5FD_ros3_fapl_ext_t ros3_fa = { + { + 1, /* Structure Version */ + FALSE, /* Authenticate? */ + "", /* AWS Region */ + "", /* Access Key ID */ + "", /* Secret Access Key */ + }, + "", /* Session/security token */ }; #endif /* H5_HAVE_ROS3_VFD */ -- cgit v0.12