From e6210c80cfe823a7ee6e938daf9e87d302ca9a8e Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:50:47 -0700 Subject: Sync ros3 VFD w/ develop (#3299) * Sync ros3 VFD w/ develop Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- src/H5FDros3.c | 300 +++++++++++++++++++++++++++++++++++++++++++++++++++--- src/H5FDros3.h | 109 +++++++++++++++++++- src/H5FDs3comms.c | 197 ++++++++++++++++++++++------------- src/H5FDs3comms.h | 3 +- test/ros3.c | 76 +++++++------- test/s3comms.c | 47 +++++---- 6 files changed, 583 insertions(+), 149 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index c874775..2208aff 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -47,6 +47,10 @@ */ static hid_t H5FD_ROS3_g = 0; +/* Session/security token property name + */ +#define ROS3_TOKEN_PROP_NAME "ros3_token_prop" + #if ROS3_STATS /* arbitrarily large value, such that any reasonable size read will be "less" @@ -566,6 +570,257 @@ H5FD__ros3_fapl_free(void *_fa) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD__ros3_fapl_free() */ +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_ros3_token + * + * Purpose: Returns session/security token of the ros3 file access + * property list though the function arguments. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token_dst /*out*/) +{ + H5P_genplist_t *plist = NULL; + char *token_src; + htri_t token_exists; + size_t tokenlen = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "izx", fapl_id, size, token_dst); + +#if ROS3_DEBUG + fprintf(stdout, "H5Pget_fapl_ros3_token() called.\n"); +#endif + + if (size == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size cannot be zero.") + if (token_dst == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "token_dst is NULL") + + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5FD_ROS3 != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") + } + + /* Copy the token data out */ + tokenlen = HDstrlen(token_src); + if (size <= tokenlen) { + tokenlen = size - 1; + } + H5MM_memcpy(token_dst, token_src, sizeof(char) * tokenlen); + token_dst[tokenlen] = '\0'; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_fapl_ros3_token() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_copy() + * + * Purpose: Create a copy of the token string. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value) +{ + char **value = (char **)_value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + +#if ROS3_DEBUG + fprintf(stdout, "H5FD__ros3_str_token_copy() called.\n"); +#endif + + if (*value) + if (NULL == (*value = HDstrdup(*value))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy string property token") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_copy() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_cmp() + * + * Purpose: Compares two token strings with each other. + * + * Return: + * - Equivalent: 0 + * - Not Equivalent: non-zero value + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static int +H5FD__ros3_str_token_cmp(const void *_value1, const void *_value2, size_t H5_ATTR_UNUSED size) +{ + char *const *value1 = (char *const *)_value1; + char *const *value2 = (char *const *)_value2; + int ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + if (*value1) { + if (*value2) + ret_value = HDstrcmp(*value1, *value2); + else + ret_value = 1; + } + else { + if (*value2) + ret_value = -1; + else + ret_value = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_cmp */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_close() + * + * Purpose: Closes/frees the memory associated to the token string. + * Currently, it is an empty implementation since there no + * additional treatment needed for this property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *_value) +{ + char **value = (char **)_value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + if (*value) + free(*value); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_close */ + +/*------------------------------------------------------------------------- + * Function: H5FD__ros3_str_token_delete() + * + * Purpose: Deletes the property token from the property list and frees + * the memory associated to the token string. + * Currently, it is an empty implementation since there no + * additional treatment needed for this property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__ros3_str_token_delete(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *_value) +{ + char **value = (char **)_value; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + if (*value) + free(*value); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__ros3_str_token_delete */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_ros3_token() + * + * Purpose: Modify the file access property list to use the H5FD_ROS3 + * driver defined in this source file by adding or + * modifying the session/security token property. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jan-Willem Blokland + * 2023-05-26 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token) +{ + H5P_genplist_t *plist = NULL; + char *token_src; + htri_t token_exists; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*s", fapl_id, token); + +#if ROS3_DEBUG + fprintf(stdout, "H5Pset_fapl_ros3_token() called.\n"); +#endif + + if (fapl_id == H5P_DEFAULT) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't set values in default property list") + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5FD_ROS3 != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if (HDstrlen(token) > H5FD_ROS3_MAX_SECRET_TOK_LEN) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, + "specified token exceeds the internally specified maximum string length") + + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "failed to check if property token exists in plist") + + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token_src) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get token value") + + memcpy(token_src, token, HDstrlen(token) + 1); + } + else { + token_src = (char *)malloc(sizeof(char) * (H5FD_ROS3_MAX_SECRET_TOK_LEN + 1)); + if (token_src == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for token_src variable."); + memcpy(token_src, token, HDstrlen(token) + 1); + if (H5P_insert(plist, ROS3_TOKEN_PROP_NAME, sizeof(char *), &token_src, NULL, NULL, NULL, NULL, + H5FD__ros3_str_token_delete, H5FD__ros3_str_token_copy, H5FD__ros3_str_token_cmp, + H5FD__ros3_str_token_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "unable to register property in plist") + } + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_ros3_token() */ + #if ROS3_STATS /*---------------------------------------------------------------------------- * @@ -613,7 +868,7 @@ ros3_reset_stats(H5FD_ros3_t *file) } done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end ros3_reset_stats() */ #endif /* ROS3_STATS */ @@ -654,6 +909,9 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) unsigned char signing_key[SHA256_DIGEST_LENGTH]; s3r_t *handle = NULL; H5FD_ros3_fapl_t fa; + H5P_genplist_t *plist = NULL; + htri_t token_exists; + char *token; H5FD_t *ret_value = NULL; FUNC_ENTER_PACKAGE @@ -681,6 +939,16 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)") + /* Session/security token */ + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, NULL, "not a file access property list") + if ((token_exists = H5P_exist_plist(plist, ROS3_TOKEN_PROP_NAME)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "failed to check if property token exists in plist") + if (token_exists) { + if (H5P_get(plist, ROS3_TOKEN_PROP_NAME, &token) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get token value") + } + /* open file; procedure depends on whether or not the fapl instructs to * authenticate requests or not. */ @@ -697,11 +965,15 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) (const char *)fa.aws_region, (const char *)iso8601now)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key") - handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, - (const unsigned char *)signing_key); + if (token_exists) + handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, + (const unsigned char *)signing_key, (const char *)token); + else + handle = H5FD_s3comms_s3r_open(url, (const char *)fa.aws_region, (const char *)fa.secret_id, + (const unsigned char *)signing_key, ""); } else - handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL, NULL); if (handle == NULL) /* If we want to check CURL's say on the matter in a controlled @@ -729,7 +1001,7 @@ done: if (ret_value == NULL) { if (handle != NULL) if (FAIL == H5FD_s3comms_s3r_close(handle)) - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle") + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle"); if (file != NULL) file = H5FL_FREE(H5FD_ros3_t, file); curl_global_cleanup(); /* early cleanup because open failed */ @@ -1011,7 +1283,7 @@ ros3_fprint_stats(FILE *stream, const H5FD_ros3_t *file) } done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* ros3_fprint_stats */ #endif /* ROS3_STATS */ @@ -1122,16 +1394,16 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) /* URL: SCHEME */ if (HDstrcmp(purl1->scheme, purl2->scheme)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); /* URL: HOST */ if (HDstrcmp(purl1->host, purl2->host)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); /* URL: PORT */ if (purl1->port && purl2->port) { if (HDstrcmp(purl1->port, purl2->port)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); } else if (purl1->port) HGOTO_DONE(-1) @@ -1141,7 +1413,7 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) /* URL: PATH */ if (purl1->path && purl2->path) { if (HDstrcmp(purl1->path, purl2->path)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); } else if (purl1->path && !purl2->path) HGOTO_DONE(-1) @@ -1151,7 +1423,7 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) /* URL: QUERY */ if (purl1->query && purl2->query) { if (HDstrcmp(purl1->query, purl2->query)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); } else if (purl1->query && !purl2->query) HGOTO_DONE(-1) @@ -1161,7 +1433,7 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) /* FAPL: AWS_REGION */ if (f1->fa.aws_region[0] != '\0' && f2->fa.aws_region[0] != '\0') { if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); } else if (f1->fa.aws_region[0] != '\0') HGOTO_DONE(-1) @@ -1171,7 +1443,7 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) /* FAPL: SECRET_ID */ if (f1->fa.secret_id[0] != '\0' && f2->fa.secret_id[0] != '\0') { if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); } else if (f1->fa.secret_id[0] != '\0') HGOTO_DONE(-1) @@ -1181,7 +1453,7 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) /* FAPL: SECRET_KEY */ if (f1->fa.secret_key[0] != '\0' && f2->fa.secret_key[0] != '\0') { if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) - HGOTO_DONE(-1) + HGOTO_DONE(-1); } else if (f1->fa.secret_key[0] != '\0') HGOTO_DONE(-1) diff --git a/src/H5FDros3.h b/src/H5FDros3.h index f84b1a4..0f2f315 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -70,12 +70,63 @@ * ****************************************************************************/ +/** + * \def H5FD_CURR_ROS3_FAPL_T_VERSION + * The version number of the H5FD_ros3_fapl_t configuration + * structure for the $H5FD_ROS3 driver. + */ #define H5FD_CURR_ROS3_FAPL_T_VERSION 1 -#define H5FD_ROS3_MAX_REGION_LEN 32 -#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +/** + * \def H5FD_ROS3_MAX_REGION_LEN + * Maximum string length for specifying the region of the S3 bucket. + */ +#define H5FD_ROS3_MAX_REGION_LEN 32 +/** + * \def H5FD_ROS3_MAX_SECRET_ID_LEN + * Maximum string length for specifying the security ID. + */ +#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +/** + * \def H5FD_ROS3_MAX_SECRET_KEY_LEN + * Maximum string length for specifying the security key. + */ #define H5FD_ROS3_MAX_SECRET_KEY_LEN 128 +/** + * \def H5FD_ROS3_MAX_SECRET_TOK_LEN + * Maximum string length for specifying the session/security token. + */ +#define H5FD_ROS3_MAX_SECRET_TOK_LEN 1024 +/** + *\struct H5FD_ros3_fapl_t + * \brief Configuration structure for H5Pset_fapl_ros3() / H5Pget_fapl_ros3(). + * + * \details H5FD_ros_fapl_t is a public structure that is used to pass + * configuration data to the #H5FD_ROS3 driver via a File Access + * Property List. A pointer to an instance of this structure is + * a parameter to H5Pset_fapl_ros3() and H5Pget_fapl_ros3(). + * + * \var int32_t H5FD_ros3_fapl_t::version + * Version number of the H5FD_ros3_fapl_t structure. Any instance passed + * to H5Pset_fapl_ros3() / H5Pget_fapl_ros3() must have a recognized version + * number or an error will be raised. Currently, this field should be set + * to #H5FD_CURR_ROS3_FAPL_T_VERSION. + * + * \var hbool_t H5FD_ros3_fapl_t::authenticate + * A boolean which specifies if security credentials should be used for + * accessing a S3 bucket. + * + * \var char H5FD_ros3_fapl_t::aws_region[H5FD_ROS3_MAX_REGION_LEN + 1] + * A string which specifies the AWS region of the S3 bucket. + * + * \var char H5FD_ros3_fapl_t::secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1] + * A string which specifies the security ID. + * + * \var char H5FD_ros3_fapl_t::secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1] + * A string which specifies the security key. + * + */ typedef struct H5FD_ros3_fapl_t { int32_t version; hbool_t authenticate; @@ -88,22 +139,72 @@ typedef struct H5FD_ros3_fapl_t { extern "C" { #endif +/** + * \brief Internal routine to initialize #H5FD_ROS3 driver. Not meant to be + * called directly by an HDF5 application. + */ H5_DLL hid_t H5FD_ros3_init(void); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Queries a File Access Property List for #H5FD_ROS3 file driver properties. + * + * \fapl_id + * \param[out] fa_out Pointer to #H5FD_ROS3 driver configuration structure. + * \returns \herr_t */ H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out); /** * \ingroup FAPL * - * \todo Add missing documentation + * \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver. + * + * \fapl_id + * \param[in] fa Pointer to #H5FD_ROS3 driver configuration structure. + * \returns \herr_t */ H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa); +/** + * \ingroup FAPL + * + * \brief Queries a File Access Property List for #H5FD_ROS3 file driver session/security + * token. + * + * \fapl_id + * \param[in] size Size of the provided char array for storing the session/security token. + * \param[out] token Session/security token. + * \returns \herr_t + * + * \since 1.14.2 + */ +H5_DLL herr_t H5Pget_fapl_ros3_token(hid_t fapl_id, size_t size, char *token); + +/** + * \ingroup FAPL + * + * \brief Modifies the specified File Access Property List to use the #H5FD_ROS3 driver + * by adding the specified session/security token. + * + * \fapl_id + * \param[in] token Session/security token. + * \returns \herr_t + * + * \details H5Pset_fapl_ros3_token() modifies an existing File Access Property List which + * is used by #H5FD_ROS3 driver by adding or updating the session/security token + * of the property list. Be aware, to set the token first you need to create + * a proper File Access Property List using H5Pset_fapl_ros() and use this list + * as input argument of the function H5Pset_fapl_ros3_token(). + * + * Note, the session token is only needed when you want to access a S3 bucket + * using temporary security credentials. + * + * \since 1.14.2 + */ +H5_DLL herr_t H5Pset_fapl_ros3_token(hid_t fapl_id, const char *token); + #ifdef __cplusplus } #endif diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index d8be2a1..5b69712 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -39,6 +39,7 @@ #include "H5Eprivate.h" /* error handling */ #include "H5MMprivate.h" /* memory management */ #include "H5FDs3comms.h" /* S3 Communications */ +#include "H5FDros3.h" /* ros3 file driver */ /****************/ /* Local Macros */ @@ -552,7 +553,7 @@ done: } } - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_s3comms_hrb_node_set() */ /*---------------------------------------------------------------------------- @@ -775,6 +776,7 @@ H5FD_s3comms_s3r_close(s3r_t *handle) H5MM_xfree(handle->secret_id); H5MM_xfree(handle->region); H5MM_xfree(handle->signing_key); + H5MM_xfree(handle->token); assert(handle->httpverb != NULL); H5MM_xfree(handle->httpverb); @@ -911,7 +913,7 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "No HTTP metadata") #if S3COMMS_DEBUG else - fprintf(stderr, "GETSIZE: OK\n"); + fprintf(stdout, "GETSIZE: OK\n"); #endif /****************** @@ -932,17 +934,21 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle) */ *end = '\0'; - content_length = HDstrtoumax((const char *)start, NULL, 0); + content_length = strtoumax((const char *)start, NULL, 0); if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t"); - if (content_length == 0 || errno == ERANGE) /* errno set by HDstrtoumax*/ + if (content_length == 0 || errno == ERANGE) /* errno set by strtoumax*/ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not convert found \"Content-Length\" response (\"%s\")", start); /* range is null-terminated, remember */ handle->filesize = (size_t)content_length; +#if S3COMMS_DEBUG + fprintf(stdout, "FILESIZE: %zu\n", handle->filesize); +#endif + /********************** * UNDO HEAD SETTINGS * **********************/ @@ -957,7 +963,7 @@ done: H5MM_xfree(headerresponse); sds.magic += 1; /* set to bad magic */ - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_s3comms_s3r_getsize */ /*---------------------------------------------------------------------------- @@ -997,7 +1003,8 @@ done: *---------------------------------------------------------------------------- */ s3r_t * -H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key) +H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key, + const char *token) { size_t tmplen = 0; CURL *curlh = NULL; @@ -1031,13 +1038,15 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const handle->region = NULL; handle->secret_id = NULL; handle->signing_key = NULL; + handle->token = NULL; handle->httpverb = NULL; /************************************* * RECORD AUTHENTICATION INFORMATION * *************************************/ - if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL)) { + if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) || + (token != NULL)) { /* if one exists, all three must exist */ if (region == NULL || region[0] == '\0') @@ -1046,6 +1055,8 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "secret id cannot be null."); if (signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "signing key cannot be null."); + if (token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "token cannot be null."); /* copy strings */ tmplen = HDstrlen(region) + 1; @@ -1065,6 +1076,12 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy."); H5MM_memcpy(handle->signing_key, signing_key, tmplen); + + tmplen = HDstrlen(token) + 1; + handle->token = (char *)H5MM_malloc(sizeof(char) * tmplen); + if (handle->token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle token copy."); + H5MM_memcpy(handle->token, token, tmplen); } /* if authentication information provided */ /************************ @@ -1121,11 +1138,12 @@ done: if (curlh != NULL) curl_easy_cleanup(curlh); if (FAIL == H5FD_s3comms_free_purl(purl)) - HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to free parsed url structure") + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to free parsed url structure"); if (handle != NULL) { H5MM_xfree(handle->region); H5MM_xfree(handle->secret_id); H5MM_xfree(handle->signing_key); + H5MM_xfree(handle->token); if (handle->httpverb != NULL) H5MM_xfree(handle->httpverb); H5MM_xfree(handle); @@ -1186,8 +1204,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) hrb_t *request = NULL; int ret = 0; /* working variable to check */ /* return value of HDsnprintf */ - struct s3r_datastruct *sds = NULL; - herr_t ret_value = SUCCEED; + char *authorization = NULL; + char *buffer1 = NULL; + char *signed_headers = NULL; + struct s3r_datastruct *sds = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -1252,6 +1273,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format HTTP Range value"); } +#if S3COMMS_DEBUG + fprintf(stdout, "%s: Bytes %" PRIuHADDR " - %" PRIuHADDR ", Request Size: %zu\n", handle->httpverb, + offset, offset + len - 1, len); +#endif + /******************* * COMPILE REQUEST * *******************/ @@ -1275,23 +1301,31 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) else { /* authenticate request */ - char authorization[512 + 1]; - /* 512 := approximate max length... - * 67 - * + 8 - * + 64 - * + 128 - * + 20 - * + 128 + authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1); + if (authorization == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for authorization variable."); + /* 2048 := approximate max length... + * 67 + * + 8 + * + 64 + * + 128 + * + 20 + * + 128 + * + 1024 */ - char buffer1[512 + 1]; /* -> Canonical Request -> Signature */ char buffer2[256 + 1]; /* -> String To Sign -> Credential */ char iso8601now[ISO8601_SIZE]; - char signed_headers[48 + 1]; + buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + + 1); /* -> Canonical Request -> Signature */ + if (buffer1 == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for buffer1 variable."); + signed_headers = (char *)H5MM_malloc(48 + H5FD_ROS3_MAX_SECRET_KEY_LEN + 1); + if (signed_headers == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for signed_headers variable."); /* should be large enough for nominal listing: - * "host;range;x-amz-content-sha256;x-amz-date" - * + '\0', with "range;" possibly absent + * "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token" + * + '\0', with "range;" and/or "x-amz-security-token" possibly absent */ /* zero start of strings */ @@ -1309,6 +1343,8 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null secret_id."); if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null signing_key."); + if (handle->token == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null token."); if (handle->httpverb == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null httpverb."); if (handle->purl->host == NULL) @@ -1340,6 +1376,15 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list."); assert(headers->magic == S3COMMS_HRB_NODE_MAGIC); + if (HDstrlen((const char *)handle->token) > 0) { + if (FAIL == + H5FD_s3comms_hrb_node_set(&headers, "x-amz-security-token", (const char *)handle->token)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header") + if (headers == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list."); + assert(headers->magic == S3COMMS_HRB_NODE_MAGIC); + } + if (rangebytesstr != NULL) { if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "Range", rangebytesstr)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set range header") @@ -1359,8 +1404,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) /**** COMPUTE AUTHORIZATION ****/ /* buffer1 -> canonical request */ - if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512, signed_headers, 48, request)) + if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + signed_headers, 48 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + request)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad canonical request"); + } /* buffer2->string-to-sign */ if (FAIL == H5FD_s3comms_tostringtosign(buffer2, buffer1, iso8601now, handle->region)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad string-to-sign"); @@ -1374,9 +1422,10 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string"); - ret = HDsnprintf(authorization, 512, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", - buffer2, signed_headers, buffer1); - if (ret <= 0 || ret >= 512) + ret = HDsnprintf(authorization, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN, + "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2, + signed_headers, buffer1); + if (ret <= 0 || ret >= 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string"); /* append authorization header to http request buffer */ @@ -1431,8 +1480,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) if (p_status != CURLE_OK) { if (CURLE_OK != curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem getting response code") - fprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %d\n", p_status, httpcode); - fprintf(stderr, "%s\n", curl_easy_strerror(p_status)); + fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %ld\n", p_status, httpcode); + fprintf(stdout, "%s\n", curl_easy_strerror(p_status)); + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "problem while performing request."); } if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL)) @@ -1447,24 +1497,36 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest) #if S3COMMS_DEBUG if (dest != NULL) { - fprintf(stderr, "len: %d\n", (int)len); - fprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n"); + fprintf(stdout, "len: %d\n", (int)len); + fprintf(stdout, "CHECKING FOR BUFFER OVERFLOW\n"); if (sds == NULL) - fprintf(stderr, "sds is NULL!\n"); + fprintf(stdout, "sds is NULL!\n"); else { - fprintf(stderr, "sds: 0x%lx\n", (long long)sds); - fprintf(stderr, "sds->size: %d\n", (int)sds->size); + fprintf(stdout, "sds: 0x%llx\n", (long long)sds); + fprintf(stdout, "sds->size: %d\n", (int)sds->size); if (len > sds->size) - fprintf(stderr, "buffer overwrite\n"); + fprintf(stdout, "buffer overwrite\n"); } } else - fprintf(stderr, "performed on entire file\n"); + fprintf(stdout, "performed on entire file\n"); #endif done: /* clean any malloc'd resources */ + if (authorization != NULL) { + H5MM_xfree(authorization); + authorization = NULL; + } + if (buffer1 != NULL) { + H5MM_xfree(buffer1); + buffer1 = NULL; + } + if (signed_headers != NULL) { + H5MM_xfree(signed_headers); + signed_headers = NULL; + } if (curlheaders != NULL) { curl_slist_free_all(curlheaders); curlheaders = NULL; @@ -1480,24 +1542,24 @@ done: if (request != NULL) { while (headers != NULL) if (FAIL == H5FD_s3comms_hrb_node_set(&headers, headers->name, NULL)) - HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot release header node") + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot release header node"); assert(NULL == headers); if (FAIL == H5FD_s3comms_hrb_destroy(&request)) - HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot release header request structure") + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot release header request structure"); assert(NULL == request); } if (curlh != NULL) { /* clear any Range */ if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL)) - HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_RANGE") + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_RANGE"); /* clear headers */ if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL)) - HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_HTTPHEADER") + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_HTTPHEADER"); } - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_s3comms_s3r_read */ /**************************************************************************** @@ -1583,8 +1645,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c size_t sh_size = (size_t)_sh_size; size_t cr_len = 0; /* working length of canonical request str */ size_t sh_len = 0; /* working length of signed headers str */ - char tmpstr[256 + 1]; - tmpstr[256] = 0; /* terminating NULL */ + char tmpstr[1024]; /* "query params" refers to the optional element in the URL, e.g. * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J @@ -1617,6 +1678,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c (size_t)3); /* three newline chars */ if (cr_len >= cr_size) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request"); + /* TODO: compiler warning */ ret = HDsnprintf(canonical_request_dest, (cr_size - 1), "%s\n%s\n%s\n", http_request->verb, http_request->resource, query_params); @@ -1629,8 +1691,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c assert(node->magic == S3COMMS_HRB_NODE_MAGIC); - ret = HDsnprintf(tmpstr, 256, "%s:%s\n", node->lowername, node->value); - if (ret < 0 || ret >= 256) + ret = HDsnprintf(tmpstr, 1024, "%s:%s\n", node->lowername, node->value); + if (ret < 0 || ret >= 1024) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s", node->lowername, node->value); cr_len += HDstrlen(tmpstr); @@ -1638,8 +1700,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request"); HDstrcat(canonical_request_dest, tmpstr); - ret = HDsnprintf(tmpstr, 256, "%s;", node->lowername); - if (ret < 0 || ret >= 256) + ret = HDsnprintf(tmpstr, 1024, "%s;", node->lowername); + if (ret < 0 || ret >= 1024) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to append semicolon to lowername %s", node->lowername); sh_len += HDstrlen(tmpstr); @@ -1663,7 +1725,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c HDstrcat(canonical_request_dest, EMPTY_SHA256); done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_s3comms_aws_canonical_request() */ /*---------------------------------------------------------------------------- @@ -1719,7 +1781,7 @@ H5FD_s3comms_bytes_to_hex(char *dest, const unsigned char *msg, size_t msg_len, } done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_s3comms_bytes_to_hex() */ /*---------------------------------------------------------------------------- @@ -1816,7 +1878,7 @@ H5FD_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *m HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not convert to hex string."); done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_s3comms_HMAC_SHA256 */ /*----------------------------------------------------------------------------- @@ -1953,7 +2015,7 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha /* "trim" tailing whitespace by replacing with null terminator*/ buffer_i = 0; - while (!HDisspace(setting_pointers[setting_i][buffer_i])) + while (!isspace(setting_pointers[setting_i][buffer_i])) buffer_i++; setting_pointers[setting_i][buffer_i] = '\0'; @@ -1963,7 +2025,7 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha } while (found_setting); done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__s3comms_load_aws_creds_from_file() */ /*---------------------------------------------------------------------------- @@ -2027,7 +2089,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * if (H5FD__s3comms_load_aws_creds_from_file(credfile, profile_name, key_id_out, secret_access_key_out, aws_region_out) == FAIL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws credentials") - if (HDfclose(credfile) == EOF) + if (fclose(credfile) == EOF) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close credentials file") credfile = NULL; } /* end if credential file opened */ @@ -2042,7 +2104,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * (*secret_access_key_out == 0) ? secret_access_key_out : NULL, (*aws_region_out == 0) ? aws_region_out : NULL) == FAIL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws config") - if (HDfclose(credfile) == EOF) + if (fclose(credfile) == EOF) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close config file") credfile = NULL; } /* end if credential file opened */ @@ -2053,10 +2115,10 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * done: if (credfile != NULL) - if (HDfclose(credfile) == EOF) - HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL, "problem error-closing aws configuration file") + if (fclose(credfile) == EOF) + HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL, "problem error-closing aws configuration file"); - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_s3comms_load_aws_profile() */ /*---------------------------------------------------------------------------- @@ -2105,7 +2167,7 @@ H5FD_s3comms_nlowercase(char *dest, const char *s, size_t len) } done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_s3comms_nlowercase() */ /*---------------------------------------------------------------------------- @@ -2147,7 +2209,7 @@ H5FD_s3comms_parse_url(const char *str, parsed_url_t **_purl) unsigned int i = 0; herr_t ret_value = FAIL; - FUNC_ENTER_NOAPI_NOINIT; + FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG printf("called H5FD_s3comms_parse_url.\n"); @@ -2181,7 +2243,7 @@ H5FD_s3comms_parse_url(const char *str, parsed_url_t **_purl) /* check for restrictions */ for (i = 0; i < len; i++) { /* scheme = [a-zA-Z+-.]+ (terminated by ":") */ - if (!HDisalpha(curstr[i]) && '+' != curstr[i] && '-' != curstr[i] && '.' != curstr[i]) + if (!isalpha(curstr[i]) && '+' != curstr[i] && '-' != curstr[i] && '.' != curstr[i]) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid SCHEME construction"); } @@ -2247,7 +2309,7 @@ H5FD_s3comms_parse_url(const char *str, parsed_url_t **_purl) else if (len > urllen) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of PORT substring"); for (i = 0; i < len; i++) - if (!HDisdigit(curstr[i])) + if (!isdigit(curstr[i])) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "PORT is not a decimal string"); /* copy port */ @@ -2310,7 +2372,7 @@ done: if (ret_value == FAIL) H5FD_s3comms_free_purl(purl); - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_s3comms_parse_url() */ /*---------------------------------------------------------------------------- @@ -2457,7 +2519,7 @@ H5FD_s3comms_percent_encode_char(char *repr, const unsigned char c, size_t *repr *(repr + *repr_len) = '\0'; done: - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_s3comms_percent_encode_char */ /*---------------------------------------------------------------------------- @@ -2547,7 +2609,7 @@ H5FD_s3comms_signing_key(unsigned char *md, const char *secret, const char *regi done: H5MM_xfree(AWS4_secret); - FUNC_LEAVE_NOAPI(ret_value); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_s3comms_signing_key() */ /*---------------------------------------------------------------------------- @@ -2694,7 +2756,7 @@ H5FD_s3comms_trim(char *dest, char *s, size_t s_len, size_t *n_written) /* Find first non-whitespace character from start; * reduce total length per character. */ - while ((s_len > 0) && HDisspace((unsigned char)s[0]) && s_len > 0) { + while ((s_len > 0) && isspace((unsigned char)s[0]) && s_len > 0) { s++; s_len--; } @@ -2706,7 +2768,7 @@ H5FD_s3comms_trim(char *dest, char *s, size_t s_len, size_t *n_written) if (s_len > 0) { do { s_len--; - } while (HDisspace((unsigned char)s[s_len])); + } while (isspace((unsigned char)s[s_len])); s_len++; /* write output into dest */ @@ -2783,8 +2845,7 @@ H5FD_s3comms_uriencode(char *dest, const char *s, size_t s_len, hbool_t encode_s */ for (s_off = 0; s_off < s_len; s_off++) { c = s[s_off]; - if (HDisalnum(c) || c == '.' || c == '-' || c == '_' || c == '~' || - (c == '/' && encode_slash == FALSE)) + if (isalnum(c) || c == '.' || c == '-' || c == '_' || c == '~' || (c == '/' && encode_slash == FALSE)) dest[dest_off++] = c; else { hex_off = 0; diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h index 1b21ad7..47857f6 100644 --- a/src/H5FDs3comms.h +++ b/src/H5FDs3comms.h @@ -478,6 +478,7 @@ typedef struct { char *region; char *secret_id; unsigned char *signing_key; + char *token; } s3r_t; #define S3COMMS_S3R_MAGIC 0x44d8d79 @@ -509,7 +510,7 @@ H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle); H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle); H5_DLL s3r_t *H5FD_s3comms_s3r_open(const char url[], const char region[], const char id[], - const unsigned char signing_key[]); + const unsigned char signing_key[], const char token[]); H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest); diff --git a/test/ros3.c b/test/ros3.c index 9a41a54..ab77d93 100644 --- a/test/ros3.c +++ b/test/ros3.c @@ -549,7 +549,7 @@ test_fapl_config_validation(void) */ success = H5Pset_fapl_ros3(fapl_id, &case_ptr->config); } - H5E_END_TRY; + H5E_END_TRY JSVERIFY(case_ptr->expected, success, case_ptr->msg) @@ -590,7 +590,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } return 1; } /* test_fapl_config_validation */ @@ -664,7 +664,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY return 1; @@ -832,7 +832,7 @@ test_vfd_open(void) { fd = H5FDopen(T.url, T.flags, _fapl_id, T.maxaddr); } - H5E_END_TRY; + H5E_END_TRY if (NULL != fd) JSVERIFY(1, 0, T.message); /* wrapper to print message and fail */ } @@ -876,14 +876,14 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } if (fapl_file_access >= 0) { H5E_BEGIN_TRY { (void)H5Pclose(fapl_file_access); } - H5E_END_TRY; + H5E_END_TRY } if (curl_ready == TRUE) { curl_global_cleanup(); @@ -1014,7 +1014,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } return 1; @@ -1078,17 +1078,17 @@ test_H5FDread_without_eoa_set_fails(void) * TEST * ********/ - H5E_BEGIN_TRY{/* mute stack trace on expected failure */ - JSVERIFY(FAIL, H5FDread(file_shakespeare, H5FD_MEM_DRAW, H5P_DEFAULT, 1200699, 102, buffer), - "cannot read before eoa is set")} H5E_END_TRY; - JSVERIFY_STR("", buffer, "buffer should remain untouched") + H5E_BEGIN_TRY{ + /* mute stack trace on expected failure */ + JSVERIFY(FAIL, H5FDread(file_shakespeare, H5FD_MEM_DRAW, H5P_DEFAULT, 1200699, 102, buffer), + "cannot read before eoa is set")} H5E_END_TRY JSVERIFY_STR("", buffer, + "buffer should remain untouched") - /************ - * TEARDOWN * - ************/ + /************ + * TEARDOWN * + ************/ - FAIL_IF(FAIL == H5FDclose(file_shakespeare)) - file_shakespeare = NULL; + FAIL_IF(FAIL == H5FDclose(file_shakespeare)) file_shakespeare = NULL; FAIL_IF(FAIL == H5Pclose(fapl_id)) fapl_id = -1; @@ -1109,7 +1109,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } return 1; @@ -1276,7 +1276,7 @@ test_read(void) { open_return = H5FDread(file_raven, H5FD_MEM_DRAW, H5P_DEFAULT, test.addr, test.len, buffer); } - H5E_END_TRY; + H5E_END_TRY JSVERIFY(test.success, open_return, test.message) if (open_return == SUCCEED) @@ -1309,7 +1309,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } return 1; @@ -1386,19 +1386,19 @@ test_noops_and_autofails(void) /* auto-fail calls to write and truncate */ H5E_BEGIN_TRY{JSVERIFY(FAIL, H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data), - "write must fail")} H5E_END_TRY; + "write must fail")} H5E_END_TRY - H5E_BEGIN_TRY{JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, FALSE), "truncate must fail")} H5E_END_TRY; + H5E_BEGIN_TRY{ + JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, FALSE), "truncate must fail")} H5E_END_TRY - H5E_BEGIN_TRY{ - JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, TRUE), "truncate must fail (closing)")} H5E_END_TRY; + H5E_BEGIN_TRY{JSVERIFY(FAIL, H5FDtruncate(file, H5P_DEFAULT, TRUE), + "truncate must fail (closing)")} H5E_END_TRY - /************ - * TEARDOWN * - ************/ + /************ + * TEARDOWN * + ************/ - FAIL_IF(FAIL == H5FDclose(file)) - file = NULL; + FAIL_IF(FAIL == H5FDclose(file)) file = NULL; FAIL_IF(FAIL == H5Pclose(fapl_id)) fapl_id = -1; @@ -1419,7 +1419,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } if (file) { (void)H5FDclose(file); @@ -1550,7 +1550,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } return 1; @@ -1620,15 +1620,15 @@ test_H5F_integration(void) /* Read-Write Open access is not allowed with this file driver. */ - H5E_BEGIN_TRY{FAIL_IF(0 <= H5Fopen(url_h5_public, H5F_ACC_RDWR, fapl_id))} H5E_END_TRY; + H5E_BEGIN_TRY{FAIL_IF(0 <= H5Fopen(url_h5_public, H5F_ACC_RDWR, fapl_id))} H5E_END_TRY - /* H5Fcreate() is not allowed with this file driver. - */ - H5E_BEGIN_TRY{FAIL_IF(0 <= H5Fcreate(url_missing, H5F_ACC_RDONLY, H5P_DEFAULT, fapl_id))} H5E_END_TRY; + /* H5Fcreate() is not allowed with this file driver. + */ + H5E_BEGIN_TRY{FAIL_IF(0 <= H5Fcreate(url_missing, H5F_ACC_RDONLY, H5P_DEFAULT, fapl_id))} H5E_END_TRY - /* Successful open. - */ - file = H5Fopen(url_h5_public, H5F_ACC_RDONLY, fapl_id); + /* Successful open. + */ + file = H5Fopen(url_h5_public, H5F_ACC_RDONLY, fapl_id); FAIL_IF(file < 0) /************ @@ -1656,7 +1656,7 @@ error: { (void)H5Pclose(fapl_id); } - H5E_END_TRY; + H5E_END_TRY } if (file > 0) (void)H5Fclose(file); diff --git a/test/s3comms.c b/test/s3comms.c index 7307673..ea32769 100644 --- a/test/s3comms.c +++ b/test/s3comms.c @@ -348,6 +348,7 @@ static int s3_test_credentials_loaded = 0; static char s3_test_aws_region[16] = ""; static char s3_test_aws_access_key_id[64] = ""; static char s3_test_aws_secret_access_key[128] = ""; +static char s3_test_aws_security_token[1024] = ""; static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; static hbool_t s3_test_bucket_defined = FALSE; @@ -1692,10 +1693,7 @@ error: } /* end test_percent_encode_char() */ /*--------------------------------------------------------------------------- - * Function: test_s3r_open() - * - * Programmer: Jacob Smith 2018-01-24 - * + * Function: test_s3r_get_filesize() *--------------------------------------------------------------------------- */ static herr_t @@ -1725,7 +1723,7 @@ test_s3r_get_filesize(void) JSVERIFY(0, H5FD_s3comms_s3r_get_filesize(NULL), "filesize of the null handle should be 0") - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL) JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) @@ -1833,14 +1831,14 @@ test_s3r_open(void) /* attempt anonymously */ - handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); /* attempt with authentication */ - handle = - H5FD_s3comms_s3r_open(url_missing, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_missing, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /************************* @@ -1849,7 +1847,7 @@ test_s3r_open(void) #if S3_TEST_RUN_TIMEOUT printf("Opening on inactive port may hang for a minute; waiting for timeout\n"); - handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); #endif @@ -1859,20 +1857,21 @@ test_s3r_open(void) /* anonymous access on restricted file */ - handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL, NULL); FAIL_IF(handle != NULL); /* passed in a bad ID */ - handle = H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID", - (const unsigned char *)signing_key); + handle = + H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID", + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /* using an invalid signing key */ - handle = - H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)EMPTY_SHA256); + handle = H5FD_s3comms_s3r_open( + url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_security_token); FAIL_IF(handle != NULL); /******************************* @@ -1881,7 +1880,7 @@ test_s3r_open(void) /* anonymous */ - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL); JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), "did not get expected filesize") JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1889,9 +1888,9 @@ test_s3r_open(void) /* using authentication on anonymously-accessible file? */ - handle = - H5FD_s3comms_s3r_open(url_raven, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_raven, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle == NULL); JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1899,9 +1898,9 @@ test_s3r_open(void) /* authenticating */ - handle = - H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, - (const char *)s3_test_aws_access_key_id, (const unsigned char *)signing_key); + handle = H5FD_s3comms_s3r_open( + url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); FAIL_IF(handle == NULL); JSVERIFY(5458199, H5FD_s3comms_s3r_get_filesize(handle), NULL) JSVERIFY(SUCCEED, H5FD_s3comms_s3r_close(handle), "unable to close file") @@ -1987,7 +1986,7 @@ test_s3r_read(void) /* open file */ - handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL, NULL); FAIL_IF(handle == NULL) JSVERIFY(6464, H5FD_s3comms_s3r_get_filesize(handle), NULL) -- cgit v0.12