ROS3: (feature) Temporary security credentials (#3030)

- Implemented support for AWS temporary security credentials.
  For this kind of credentials also a session/security token
  should be included in the request by adding the
  x-amz-security-token header.

Co-authored-by: Larry Knox <lrknox@hdfgroup.org>
Co-authored-by: Jordan Henderson <jhenderson@hdfgroup.org>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

1 files changed, 81 insertions, 28 deletions

diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c
index 74a4bbc..10b3ff2 100644
--- a/src/H5FDs3comms.c
+++ b/src/H5FDs3comms.c
@@ -39,6 +39,7 @@
 #include "H5Eprivate.h"  /* error handling    */
 #include "H5MMprivate.h" /* memory management */
 #include "H5FDs3comms.h" /* S3 Communications */
+#include "H5FDros3.h"    /* ros3 file driver  */
 
 /****************/
 /* Local Macros */
@@ -775,6 +776,7 @@ H5FD_s3comms_s3r_close(s3r_t *handle)
     H5MM_xfree(handle->secret_id);
     H5MM_xfree(handle->region);
     H5MM_xfree(handle->signing_key);
+    H5MM_xfree(handle->token);
 
     assert(handle->httpverb != NULL);
     H5MM_xfree(handle->httpverb);
@@ -1001,7 +1003,8 @@ done:
  *----------------------------------------------------------------------------
  */
 s3r_t *
-H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key)
+H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const unsigned char *signing_key,
+                      const char *token)
 {
     size_t        tmplen    = 0;
     CURL         *curlh     = NULL;
@@ -1035,13 +1038,15 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
     handle->region      = NULL;
     handle->secret_id   = NULL;
     handle->signing_key = NULL;
+    handle->token       = NULL;
     handle->httpverb    = NULL;
 
     /*************************************
      * RECORD AUTHENTICATION INFORMATION *
      *************************************/
 
-    if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL)) {
+    if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) ||
+        (token != NULL)) {
 
         /* if one exists, all three must exist */
         if (region == NULL || region[0] == '\0')
@@ -1050,6 +1055,8 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "secret id cannot be null.");
         if (signing_key == NULL)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "signing key cannot be null.");
+        if (token == NULL)
+            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "token cannot be null.");
 
         /* copy strings */
         tmplen         = HDstrlen(region) + 1;
@@ -1069,6 +1076,12 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
         if (handle->signing_key == NULL)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy.");
         H5MM_memcpy(handle->signing_key, signing_key, tmplen);
+
+        tmplen        = HDstrlen(token) + 1;
+        handle->token = (char *)H5MM_malloc(sizeof(char) * tmplen);
+        if (handle->token == NULL)
+            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle token copy.");
+        H5MM_memcpy(handle->token, token, tmplen);
     } /* if authentication information provided */
 
     /************************
@@ -1130,6 +1143,7 @@ done:
             H5MM_xfree(handle->region);
             H5MM_xfree(handle->secret_id);
             H5MM_xfree(handle->signing_key);
+            H5MM_xfree(handle->token);
             if (handle->httpverb != NULL)
                 H5MM_xfree(handle->httpverb);
             H5MM_xfree(handle);
@@ -1190,8 +1204,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
     hrb_t             *request       = NULL;
     int                ret           = 0; /* working variable to check  */
                                           /* return value of HDsnprintf  */
-    struct s3r_datastruct *sds       = NULL;
-    herr_t                 ret_value = SUCCEED;
+    char                  *authorization  = NULL;
+    char                  *buffer1        = NULL;
+    char                  *signed_headers = NULL;
+    struct s3r_datastruct *sds            = NULL;
+    herr_t                 ret_value      = SUCCEED;
 
     FUNC_ENTER_NOAPI_NOINIT
 
@@ -1284,23 +1301,31 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
     else {
         /* authenticate request
          */
-        char authorization[512 + 1];
-        /*   512 := approximate max length...
-         *    67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request,"
-         *           "SignedHeaders=,Signature=")>
-         * +   8 <yyyyMMDD>
-         * +  64 <hex(sha256())>
-         * + 128 <max? len(secret_id)>
-         * +  20 <max? len(region)>
-         * + 128 <max? len(signed_headers)>
+        authorization = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN + 1);
+        if (authorization == NULL)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for authorization variable.");
+        /*   2048 := approximate max length...
+         *     67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request,"
+         *             "SignedHeaders=,Signature=")>
+         * +    8 <yyyyMMDD>
+         * +   64 <hex(sha256())>
+         * +  128 <max? len(secret_id)>
+         * +   20 <max? len(region)>
+         * +  128 <max? len(signed_headers)>
+         * + 1024 <max? len(session_token)>
          */
-        char buffer1[512 + 1]; /* -> Canonical Request -> Signature */
         char buffer2[256 + 1]; /* -> String To Sign -> Credential */
         char iso8601now[ISO8601_SIZE];
-        char signed_headers[48 + 1];
+        buffer1 = (char *)H5MM_malloc(512 + H5FD_ROS3_MAX_SECRET_TOK_LEN +
+                                      1); /* -> Canonical Request -> Signature */
+        if (buffer1 == NULL)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for buffer1 variable.");
+        signed_headers = (char *)H5MM_malloc(48 + H5FD_ROS3_MAX_SECRET_KEY_LEN + 1);
+        if (signed_headers == NULL)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for signed_headers variable.");
         /* should be large enough for nominal listing:
-         * "host;range;x-amz-content-sha256;x-amz-date"
-         * + '\0', with "range;" possibly absent
+         * "host;range;x-amz-content-sha256;x-amz-date;x-amz-security-token"
+         * + '\0', with "range;" and/or "x-amz-security-token" possibly absent
          */
 
         /* zero start of strings */
@@ -1318,6 +1343,8 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null secret_id.");
         if (handle->signing_key == NULL)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null signing_key.");
+        if (handle->token == NULL)
+            HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null token.");
         if (handle->httpverb == NULL)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle must have non-null httpverb.");
         if (handle->purl->host == NULL)
@@ -1349,6 +1376,15 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
         assert(headers->magic == S3COMMS_HRB_NODE_MAGIC);
 
+        if (HDstrlen((const char *)handle->token) > 0) {
+            if (FAIL ==
+                H5FD_s3comms_hrb_node_set(&headers, "x-amz-security-token", (const char *)handle->token))
+                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set x-amz-security-token header")
+            if (headers == NULL)
+                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem building headers list.");
+            assert(headers->magic == S3COMMS_HRB_NODE_MAGIC);
+        }
+
         if (rangebytesstr != NULL) {
             if (FAIL == H5FD_s3comms_hrb_node_set(&headers, "Range", rangebytesstr))
                 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set range header")
@@ -1368,8 +1404,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
         /**** COMPUTE AUTHORIZATION ****/
 
         /* buffer1 -> canonical request */
-        if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512, signed_headers, 48, request))
+        if (FAIL == H5FD_s3comms_aws_canonical_request(buffer1, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
+                                                       signed_headers, 48 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
+                                                       request)) {
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad canonical request");
+        }
         /* buffer2->string-to-sign */
         if (FAIL == H5FD_s3comms_tostringtosign(buffer2, buffer1, iso8601now, handle->region))
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad string-to-sign");
@@ -1383,9 +1422,10 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
         if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string");
 
-        ret = HDsnprintf(authorization, 512, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s",
-                         buffer2, signed_headers, buffer1);
-        if (ret <= 0 || ret >= 512)
+        ret = HDsnprintf(authorization, 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN,
+                         "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2,
+                         signed_headers, buffer1);
+        if (ret <= 0 || ret >= 512 + H5FD_ROS3_MAX_SECRET_TOK_LEN)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string");
 
         /* append authorization header to http request buffer */
@@ -1440,8 +1480,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
         if (p_status != CURLE_OK) {
             if (CURLE_OK != curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode))
                 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem getting response code")
-            fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %d\n", p_status, httpcode);
+            fprintf(stdout, "CURL ERROR CODE: %d\nHTTP CODE: %ld\n", p_status, httpcode);
             fprintf(stdout, "%s\n", curl_easy_strerror(p_status));
+
             HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, "problem while performing request.");
         }
         if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL))
@@ -1474,6 +1515,18 @@ H5FD_s3comms_s3r_read(s3r_t *handle, haddr_t offset, size_t len, void *dest)
 done:
     /* clean any malloc'd resources
      */
+    if (authorization != NULL) {
+        H5MM_xfree(authorization);
+        authorization = NULL;
+    }
+    if (buffer1 != NULL) {
+        H5MM_xfree(buffer1);
+        buffer1 = NULL;
+    }
+    if (signed_headers != NULL) {
+        H5MM_xfree(signed_headers);
+        signed_headers = NULL;
+    }
     if (curlheaders != NULL) {
         curl_slist_free_all(curlheaders);
         curlheaders = NULL;
@@ -1592,8 +1645,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
     size_t      sh_size      = (size_t)_sh_size;
     size_t      cr_len       = 0; /* working length of canonical request str */
     size_t      sh_len       = 0; /* working length of signed headers str */
-    char        tmpstr[256 + 1];
-    tmpstr[256] = 0; /* terminating NULL */
+    char        tmpstr[1024];
 
     /* "query params" refers to the optional element in the URL, e.g.
      *     http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J
@@ -1626,6 +1678,7 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
               (size_t)3); /* three newline chars */
     if (cr_len >= cr_size)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request");
+
     /* TODO: compiler warning */
     ret = HDsnprintf(canonical_request_dest, (cr_size - 1), "%s\n%s\n%s\n", http_request->verb,
                      http_request->resource, query_params);
@@ -1638,8 +1691,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
 
         assert(node->magic == S3COMMS_HRB_NODE_MAGIC);
 
-        ret = HDsnprintf(tmpstr, 256, "%s:%s\n", node->lowername, node->value);
-        if (ret < 0 || ret >= 256)
+        ret = HDsnprintf(tmpstr, 1024, "%s:%s\n", node->lowername, node->value);
+        if (ret < 0 || ret >= 1024)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s",
                         node->lowername, node->value);
         cr_len += HDstrlen(tmpstr);
@@ -1647,8 +1700,8 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, int _cr_size, c
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not enough space in canonical request");
         HDstrcat(canonical_request_dest, tmpstr);
 
-        ret = HDsnprintf(tmpstr, 256, "%s;", node->lowername);
-        if (ret < 0 || ret >= 256)
+        ret = HDsnprintf(tmpstr, 1024, "%s;", node->lowername);
+        if (ret < 0 || ret >= 1024)
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to append semicolon to lowername %s",
                         node->lowername);
         sh_len += HDstrlen(tmpstr);