summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorQuincey Koziol <koziol@koziol.gov>2019-08-21 20:36:17 (GMT)
committerQuincey Koziol <koziol@koziol.gov>2019-08-21 20:36:17 (GMT)
commit5d6f27b32be6919ab64046bd8ff27c69d84a2c17 (patch)
tree3774e701d57114cafa7bc4ab400e0fdb054472e6 /src
parent0b85fa6004cb96d222329614fa446b98cfcc06ea (diff)
parentf0272e4945f047f17043b484db594df131222f1e (diff)
downloadhdf5-5d6f27b32be6919ab64046bd8ff27c69d84a2c17.zip
hdf5-5d6f27b32be6919ab64046bd8ff27c69d84a2c17.tar.gz
hdf5-5d6f27b32be6919ab64046bd8ff27c69d84a2c17.tar.bz2
Merge branch 'develop' into shared_file_ptr
Diffstat (limited to 'src')
-rw-r--r--src/H5FDros3.c111
-rw-r--r--src/H5FDros3.h42
-rw-r--r--src/H5FDs3comms.c336
-rw-r--r--src/H5FDs3comms.h83
-rw-r--r--src/Makefile.am10
5 files changed, 65 insertions, 517 deletions
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
index 18f1f91..a369ca2 100644
--- a/src/H5FDros3.c
+++ b/src/H5FDros3.c
@@ -34,6 +34,8 @@
#include "H5MMprivate.h" /* Memory management */
#include "H5FDs3comms.h" /* S3 Communications */
+#ifdef H5_HAVE_ROS3_VFD
+
/* toggle function call prints: 1 turns on
*/
#define ROS3_DEBUG 0
@@ -119,8 +121,6 @@ static unsigned long long ros3_stats_boundaries[ROS3_STATS_BIN_COUNT];
*
* Programmer: Jacob Smith
*
- * Changes: None
- *
***************************************************************************/
typedef struct {
unsigned long long count;
@@ -190,8 +190,6 @@ typedef struct {
*
* Programmer: Jacob Smith
*
- * Changes: None.
- *
***************************************************************************/
typedef struct H5FD_ros3_t {
H5FD_t pub;
@@ -215,9 +213,7 @@ typedef struct H5FD_ros3_t {
*
*/
#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1)
-#ifdef H5_HAVE_ROS3_VFD
#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR))
-#endif /* H5_HAVE_ROS3_VFD */
/* Prototypes */
static herr_t H5FD_ros3_term(void);
@@ -279,10 +275,8 @@ static const H5FD_class_t H5FD_ros3_g = {
H5FD_FLMAP_DICHOTOMY /* fl_map */
};
-#ifdef H5_HAVE_ROS3_VFD
/* Declare a free list to manage the H5FD_ros3_t struct */
H5FL_DEFINE_STATIC(H5FD_ros3_t);
-#endif /* H5_HAVE_ROS3_VFD */
/*-------------------------------------------------------------------------
@@ -292,8 +286,7 @@ H5FL_DEFINE_STATIC(H5FD_ros3_t);
*
* Return: Non-negative on success/Negative on failure
*
- * Changes: Rename as appropriate for ros3 vfd.
- * Jacob Smith 2017
+ * Programmer: Jacob Smith 2017
*
*-------------------------------------------------------------------------
*/
@@ -324,11 +317,7 @@ done:
* Return: Success: The driver ID for the ros3 driver.
* Failure: Negative
*
- * Programmer: Robb Matzke
- * Thursday, July 29, 1999
- *
- * Changes: Rename as appropriate for ros3 vfd.
- * Jacob Smith 2017
+ * Programmer: Jacob Smith 2017
*
*-------------------------------------------------------------------------
*/
@@ -372,11 +361,7 @@ done:
*
* Returns: SUCCEED (Can't fail)
*
- * Programmer: Quincey Koziol
- * Friday, Jan 30, 2004
- *
- * Changes: Rename as appropriate for ros3 vfd.
- * Jacob Smith 2017
+ * Programmer: Jacob Smith 2017
*
*---------------------------------------------------------------------------
*/
@@ -464,10 +449,6 @@ done:
* Programmer: Jacob Smith
* 9/10/17
*
- * Changes: Add checks for authenticate flag requring populated
- * `aws_region` and `secret_id` strings.
- * -- Jacob Smith 2017-11-01
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -479,7 +460,7 @@ H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa)
HDassert(fa != NULL);
- if ( fa->version != H5FD__CURR_ROS3_FAPL_T_VERSION ) {
+ if ( fa->version != H5FD_CURR_ROS3_FAPL_T_VERSION ) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"Unknown H5FD_ros3_fapl_t version");
}
@@ -704,8 +685,6 @@ H5FD_ros3_fapl_free(void *_fa)
* Programmer: Jacob Smith
* 2017-12-08
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
static herr_t
@@ -774,12 +753,8 @@ done:
* Programmer: Jacob Smith
* 2017-11-02
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
-#ifdef H5_HAVE_ROS3_VFD
-
static H5FD_t *
H5FD_ros3_open(
const char *url,
@@ -904,22 +879,6 @@ done:
} /* end H5FD_ros3_open() */
-#else /* H5_HAVE_ROS3_VFD not defined */
-
-static H5FD_t *
-H5FD_ros3_open(
- const char H5_ATTR_UNUSED *url,
- unsigned H5_ATTR_UNUSED flags,
- hid_t H5_ATTR_UNUSED fapl_id,
- haddr_t H5_ATTR_UNUSED maxaddr)
-{
- H5FD_t *ret_value = NULL;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_ros3_open() */
-
-#endif /* H5_HAVE_ROS3_VFD */
-
#if ROS3_STATS
/*----------------------------------------------------------------------------
@@ -974,8 +933,6 @@ H5FD_ros3_open(
*
* Programmer: Jacob Smith
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
static herr_t
@@ -1228,12 +1185,8 @@ done:
* Programmer: Jacob Smith
* 2017-11-02
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
-#ifdef H5_HAVE_ROS3_VFD
-
static herr_t
H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
{
@@ -1277,18 +1230,6 @@ done:
} /* end H5FD_ros3_close() */
-#else /* H5_HAVE_ROS3_VFD not defined */
-
-static herr_t
-H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
-{
- herr_t ret_value = FAIL;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_ros3_close() */
-
-#endif /* H5_HAVE_ROS3_VFD */
-
/*-------------------------------------------------------------------------
*
@@ -1318,17 +1259,8 @@ H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
* Programmer: Jacob Smith
* 2017-11-06
*
- * Changes:
- *
- * + Change from strcmp-like return values (-1, 0, 1) to instead return
- * binary equivalence (0) or inequality (-1).
- * + Replace "if still equal then check this" waterfall with GOTO jumps.
- * Jacob Smith 2018-05-17
- *
*-------------------------------------------------------------------------
*/
-#ifdef H5_HAVE_ROS3_VFD
-
static int
H5FD_ros3_cmp(
const H5FD_t *_f1,
@@ -1463,20 +1395,6 @@ done:
} /* H5FD_ros3_cmp() */
-#else /* H5_HAVE_ROS3_VFD not defined */
-
-static int
-H5FD_ros3_cmp(
- const H5FD_t H5_ATTR_UNUSED *_f1,
- const H5FD_t H5_ATTR_UNUSED *_f2)
-{
- int ret_value = 0;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value)
-} /* H5FD_ros3_cmp() */
-
-#endif /* H5_HAVE_ROS3_VFD */
-
/*-------------------------------------------------------------------------
* Function: H5FD_ros3_query
@@ -1536,8 +1454,6 @@ H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file,
* Programmer: Jacob Smith
* 2017-11-02
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static haddr_t
@@ -1572,8 +1488,6 @@ H5FD_ros3_get_eoa(const H5FD_t *_file,
* Programmer: Jacob Smith
* 2017-11-03
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1646,8 +1560,6 @@ H5FD_ros3_get_eof(const H5FD_t *_file,
* Programmer: Jacob Smith
* 2017-11-02
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1696,8 +1608,6 @@ done:
* Programmer: Jacob Smith
* 2017-11-??
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1792,8 +1702,6 @@ done:
* Programmer: Jacob Smith
* 2017-10-23
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1839,8 +1747,6 @@ done:
* Programmer: Jacob Smith
* 2017-10-23
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1884,8 +1790,6 @@ done:
* Programmer: Jacob Smith
* 2017-11-03
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1914,8 +1818,6 @@ H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file,
* Programmer: Jacob Smith
* 2017-11-03
*
- * Changes: None.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1926,4 +1828,5 @@ H5FD_ros3_unlock(H5FD_t H5_ATTR_UNUSED *_file)
} /* end H5FD_ros3_unlock() */
+#endif /* H5_HAVE_ROS3_VFD */
diff --git a/src/H5FDros3.h b/src/H5FDros3.h
index 7dfc95b..250c7cc 100644
--- a/src/H5FDros3.h
+++ b/src/H5FDros3.h
@@ -21,11 +21,13 @@
#ifndef H5FDros3_H
#define H5FDros3_H
-#define H5FD_ROS3 (H5FD_ros3_init())
+#ifdef H5_HAVE_ROS3_VFD
+#define H5FD_ROS3 (H5FD_ros3_init())
+#else
+#define H5FD_ROS3 (H5I_INVALID_HID)
+#endif /* H5_HAVE_ROS3_VFD */
-#ifdef __cplusplus
-extern "C" {
-#endif
+#ifdef H5_HAVE_ROS3_VFD
/****************************************************************************
*
@@ -46,7 +48,7 @@ extern "C" {
* to the above calls must have a recognized version number, or an error
* will be flagged.
*
- * This field should be set to H5FD__CURR_ROS3_FAPL_T_VERSION.
+ * This field should be set to H5FD_CURR_ROS3_FAPL_T_VERSION.
*
* `authenticate` (hbool_t)
*
@@ -67,31 +69,27 @@ extern "C" {
*
* String: "Secret Access Key" associated with the ID and resource.
*
- *
- *
- * Programmer: John Mainzer
- *
- * Changes:
- *
- * - Add documentation of fields (except `version`)
- * --- Jacob Smith 2017-12-04
- *
****************************************************************************/
-#define H5FD__CURR_ROS3_FAPL_T_VERSION 1
+#define H5FD_CURR_ROS3_FAPL_T_VERSION 1
-#define H5FD__ROS3_MAX_REGION_LEN 32
-#define H5FD__ROS3_MAX_SECRET_ID_LEN 128
-#define H5FD__ROS3_MAX_SECRET_KEY_LEN 128
+#define H5FD_ROS3_MAX_REGION_LEN 32
+#define H5FD_ROS3_MAX_SECRET_ID_LEN 128
+#define H5FD_ROS3_MAX_SECRET_KEY_LEN 128
typedef struct H5FD_ros3_fapl_t {
int32_t version;
hbool_t authenticate;
- char aws_region[H5FD__ROS3_MAX_REGION_LEN + 1];
- char secret_id[H5FD__ROS3_MAX_SECRET_ID_LEN + 1];
- char secret_key[H5FD__ROS3_MAX_SECRET_KEY_LEN + 1];
+ char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1];
+ char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1];
+ char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1];
} H5FD_ros3_fapl_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
H5_DLL hid_t H5FD_ros3_init(void);
H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t * fa_out);
H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t * fa);
@@ -100,6 +98,8 @@ H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t * fa);
}
#endif
+#endif /* H5_HAVE_ROS3_VFD */
+
#endif /* ifndef H5FDros3_H */
diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c
index 42b281f..f8b4417 100644
--- a/src/H5FDs3comms.c
+++ b/src/H5FDs3comms.c
@@ -47,12 +47,12 @@
/* Local Macros */
/****************/
+#ifdef H5_HAVE_ROS3_VFD
+
/* toggle debugging (enable with 1)
*/
#define S3COMMS_DEBUG 0
-#ifdef H5_HAVE_ROS3_VFD
-
/* manipulate verbosity of CURL output
* operates separately from S3COMMS_DEBUG
*
@@ -67,7 +67,6 @@
*/
#define S3COMMS_MAX_RANGE_STRING_SIZE 128
-#endif /* H5_HAVE_ROS3_VFD */
/******************/
/* Local Typedefs */
@@ -137,8 +136,6 @@ herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle);
* Programmer: Jacob Smith
* 2017-08-17
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
size_t
@@ -217,14 +214,6 @@ curlwritecallback(char *ptr,
* Programmer: Jacob Smith
* 2017-09-22
*
- * Changes:
- *
- * - Change return value to herr_t
- * - Change list pointer to pointer-to-pointer-to-node
- * - Change to use singly-linked list (from twin doubly-linked lists)
- * with modification to hrb_node_t
- * --- Jake Smith 2017-01-17
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -635,19 +624,6 @@ done:
* Programmer: Jacob Smith
* 2017-07-21
*
- * Changes:
- *
- * - Conditional free() of `hrb_node_t` pointer properties based on
- * `which_free` property.
- * --- Jacob Smith 2017-08-08
- *
- * - Integrate with HDF5.
- * - Returns herr_t instead of nothing.
- * --- Jacob Smith 2017-09-21
- *
- * - Change argument to from *buf to **buf, to null pointer within call
- * --- Jacob Smith 2017-20-05
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -709,25 +685,6 @@ done:
* Programmer: Jacob Smith
* 2017-07-21
*
- * Changes:
- *
- * - Update struct membership for newer 'generic' `hrb_t` format.
- * --- Jacob Smith, 2017-07-24
- *
- * - Rename from `hrb_new()` to `hrb_request()`
- * --- Jacob Smith, 2017-07-25
- *
- * - Integrate with HDF5.
- * - Rename from 'hrb_request()` to `H5FD_s3comms_hrb_init_request()`.
- * - Remove `host` from input parameters.
- * - Host, as with all other fields, must now be added through the
- * add-field functions.
- * - Add `version` (HTTP version string, e.g. "HTTP/1.1") to parameters.
- * --- Jacob Smith 2017-09-20
- *
- * - Update to use linked-list `hrb_node_t` headers in structure.
- * --- Jacob Smith 2017-10-05
- *
*----------------------------------------------------------------------------
*/
hrb_t *
@@ -842,64 +799,10 @@ done:
} /* end H5FD_s3comms_hrb_init_request() */
-
/****************************************************************************
* S3R FUNCTIONS
****************************************************************************/
-#ifndef H5_HAVE_ROS3_VFD
-/* No-op, auto-fail, unused-variable implementations if ROS3 is not enabled */
-
-herr_t
-H5FD_s3comms_s3r_close(s3r_t H5_ATTR_UNUSED *handle)
-{
- herr_t ret_value = SUCCEED;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_s3r_close() */
-
-size_t
-H5FD_s3comms_s3r_get_filesize(s3r_t H5_ATTR_UNUSED *handle)
-{
- size_t ret_value = 0;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_s3r_get_filesize() */
-
-herr_t
-H5FD_s3comms_s3r_getsize(s3r_t H5_ATTR_UNUSED *handle)
-{
- herr_t ret_value = FAIL;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value);
-} /* end H5FD_s3comms_s3r_getsize() */
-
-s3r_t *
-H5FD_s3comms_s3r_open(
- const char H5_ATTR_UNUSED *url,
- const char H5_ATTR_UNUSED *region,
- const char H5_ATTR_UNUSED *id,
- const unsigned char H5_ATTR_UNUSED *signing_key)
-{
- s3r_t *ret_value = NULL;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_s3comms_s3r_open() */
-
-herr_t
-H5FD_s3comms_s3r_read(
- s3r_t H5_ATTR_UNUSED *handle,
- haddr_t H5_ATTR_UNUSED offset,
- size_t H5_ATTR_UNUSED len,
- void H5_ATTR_UNUSED *dest)
-{
- herr_t ret_value = FAIL;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value);
-} /* end H5FD_s3comms_s3r_read() */
-
-#else
-
/*----------------------------------------------------------------------------
*
@@ -920,20 +823,6 @@ H5FD_s3comms_s3r_read(
* Programmer: Jacob Smith
* 2017-08-31
*
- * Changes:
- *
- * - Remove all messiness related to the now-gone "setopt" utility
- * as it no longer exists in the handle.
- * - Return type to `void`.
- * --- Jacob Smith 2017-09-01
- *
- * - Incorporate into HDF environment.
- * - Rename from `s3r_close()` to `H5FD_s3comms_s3r_close()`.
- * --- Jacob Smith 2017-10-06
- *
- * - Change separate host, resource, port info to `parsed_url_t` struct ptr.
- * --- Jacob Smith 2017-11-01
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -995,8 +884,6 @@ done:
*
* Programmer: Jacob Smith 2017-01-14
*
- * Changes: None
- *
*----------------------------------------------------------------------------
*/
size_t
@@ -1044,44 +931,24 @@ H5FD_s3comms_s3r_get_filesize(s3r_t *handle)
* Programmer: Jacob Smith
* 2017-08-23
*
- * Changes:
- *
- * - Update to revised `s3r_t` format and life cycle.
- * --- Jacob Smith 2017-09-01
- *
- * - Conditional change to static header buffer and structure.
- * --- Jacob Smith 2017-09-05
- *
- * - Incorporate into HDF environment.
- * - Rename from `s3r_getsize()` to `H5FD_s3comms_s3r_getsize()`.
- * --- Jacob Smith 2017-10-06
- *
*----------------------------------------------------------------------------
*/
herr_t
H5FD_s3comms_s3r_getsize(s3r_t *handle)
{
-#ifdef H5_HAVE_ROS3_VFD
uintmax_t content_length = 0;
CURL *curlh = NULL;
char *end = NULL;
char *headerresponse = NULL;
- herr_t ret_value = SUCCEED;
struct s3r_datastruct sds = {
S3COMMS_CALLBACK_DATASTRUCT_MAGIC,
NULL,
0 };
char *start = NULL;
-#else
- herr_t ret_value = FAIL;
-#endif /* H5_HAVE_ROS3_VFD */
-
-
+ herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
-#ifdef H5_HAVE_ROS3_VFD
-
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_s3r_getsize.\n");
#endif
@@ -1238,8 +1105,6 @@ done:
H5MM_xfree(headerresponse);
sds.magic += 1; /* set to bad magic */
-#endif /* H5_HAVE_ROS3_VFD */
-
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD_s3comms_s3r_getsize */
@@ -1282,20 +1147,6 @@ done:
* Programmer: Jacob Smith
* 2017-09-01
*
- * Changes:
- *
- * - Incorporate into HDF environment.
- * - Rename from `s3r_open()` to `H5FD_s3comms_s3r_open()`.
- * --- Jacob Smith 2017-10-06
- *
- * - Remove port number from signature.
- * - Name (`url`) must be complete url with http scheme and optional port
- * number in string.
- * - e.g., "http://bucket.aws.com:9000/myfile.dat?query=param"
- * - Internal storage of host, resource, and port information moved into
- * `parsed_url_t` struct pointer.
- * --- Jacob Smith 2017-11-01
- *
*----------------------------------------------------------------------------
*/
s3r_t *
@@ -1304,20 +1155,14 @@ H5FD_s3comms_s3r_open(const char *url,
const char *id,
const unsigned char *signing_key)
{
-#ifdef H5_HAVE_ROS3_VFD
size_t tmplen = 0;
CURL *curlh = NULL;
s3r_t *handle = NULL;
parsed_url_t *purl = NULL;
-#endif
s3r_t *ret_value = NULL;
-
-
FUNC_ENTER_NOAPI_NOINIT
-#ifdef H5_HAVE_ROS3_VFD
-
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_s3r_open.\n");
#endif
@@ -1492,11 +1337,9 @@ H5FD_s3comms_s3r_open(const char *url,
HDmemcpy(handle->httpverb, "GET", 4);
ret_value = handle;
-#endif /* H5_HAVE_ROS3_VFD */
done:
if (ret_value == NULL) {
-#ifdef H5_HAVE_ROS3_VFD
if (curlh != NULL) {
curl_easy_cleanup(curlh);
}
@@ -1513,7 +1356,6 @@ done:
}
H5MM_xfree(handle);
}
-#endif /* H5_HAVE_ROS3_VFD */
}
FUNC_LEAVE_NOAPI(ret_value)
@@ -1560,29 +1402,6 @@ done:
* Programmer: Jacob Smith
* 2017-08-22
*
- * Changes:
- *
- * - Revise structure to prevent unnecessary hrb_t element creation.
- * - Rename tmprstr -> rangebytesstr to reflect purpose.
- * - Insert needed `free()`s, particularly for `sds`.
- * --- Jacob Smith 2017-08-23
- *
- * - Revise heavily to accept buffer, range as parameters.
- * - Utilize modified s3r_t format.
- * --- Jacob Smith 2017-08-31
- *
- * - Incorporate into HDF library.
- * - Rename from `s3r_read()` to `H5FD_s3comms_s3r_read()`.
- * - Return `herr_t` succeed/fail instead of S3code.
- * - Update to use revised `hrb_t` and `hrb_node_t` structures.
- * --- Jacob Smith 2017-10-06
- *
- * - Update to use `parsed_url_t *purl` in handle.
- * --- Jacob Smith 2017-11-01
- *
- * - Better define behavior upon read past EOF
- * --- Jacob Smith 2017-01-19
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -1591,7 +1410,6 @@ H5FD_s3comms_s3r_read(s3r_t *handle,
size_t len,
void *dest)
{
-#ifdef H5_HAVE_ROS3_VFD
CURL *curlh = NULL;
CURLcode p_status = CURLE_OK;
struct curl_slist *curlheaders = NULL;
@@ -1604,16 +1422,9 @@ H5FD_s3comms_s3r_read(s3r_t *handle,
/* return value of HDsnprintf */
struct s3r_datastruct *sds = NULL;
herr_t ret_value = SUCCEED;
-#else
- herr_t ret_value = FAIL;
-#endif /* H5_HAVE_ROS3_VFD */
-
-
FUNC_ENTER_NOAPI_NOINIT
-#ifdef H5_HAVE_ROS3_VFD
-
#if S3COMMS_DEBUG
HDfprintf(stdout, "called H5FD_s3comms_s3r_read.\n");
#endif
@@ -2105,12 +1916,9 @@ done:
}
}
-#endif /* H5_HAVE_ROS3_VFD */
-
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD_s3comms_s3r_read */
-#endif /* H5_HAVE_ROS3_VFD */
/****************************************************************************
* MISCELLANEOUS FUNCTIONS
@@ -2133,8 +1941,6 @@ done:
* Programmer: Jacob Smith
* 2017-07-12
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
struct tm *
@@ -2192,8 +1998,6 @@ gmnow(void)
* Programmer: Jacob Smith
* 2017-10-04
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -2362,20 +2166,6 @@ done:
* Programmer: Jacob Smith
* 2017-07-12
*
- * Changes:
- *
- * - Integrate into HDF.
- * - Rename from hex() to H5FD_s3comms_bytes_to_hex.
- * - Change return type from `void` to `herr_t`.
- * --- Jacob Smtih 2017-09-14
- *
- * - Add bool parameter `lowercase` to configure upper/lowercase output
- * of a-f hex characters.
- * --- Jacob Smith 2017-09-19
- *
- * - Change bool type to `hbool_t`
- * --- Jacob Smtih 2017-10-11
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -2438,8 +2228,6 @@ done:
* Programmer: Jacob Smith
* 2017-11-01
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -2494,22 +2282,8 @@ H5FD_s3comms_free_purl(parsed_url_t *purl)
* Programmer: Jacob Smith
* 2017-07-??
*
- * Changes:
- *
- * - Integrate with HDF5.
- * - Rename from `HMAC_SHA256` to `H5FD_s3comms_HMAC_SHA256`.
- * - Rename output parameter from `md` to `dest`.
- * - Return `herr_t` type instead of `void`.
- * - Call `H5FD_s3comms_bytes_to_hex` to generate hex cleartext for output.
- * --- Jacob Smith 2017-09-19
- *
- * - Use static char array instead of malloc'ing `md`
- * --- Jacob Smith 2017-10-10
- *
*----------------------------------------------------------------------------
*/
-#ifdef H5_HAVE_ROS3_VFD
-
herr_t
H5FD_s3comms_HMAC_SHA256(
const unsigned char *key,
@@ -2556,23 +2330,6 @@ done:
FUNC_LEAVE_NOAPI(ret_value);
} /* H5FD_s3comms_HMAC_SHA256 */
-#else /* H5_HAVE_ROS3_VFD not defined */
-
-herr_t
-H5FD_s3comms_HMAC_SHA256(
- const unsigned char H5_ATTR_UNUSED *key,
- size_t H5_ATTR_UNUSED key_len,
- const char H5_ATTR_UNUSED *msg,
- size_t H5_ATTR_UNUSED msg_len,
- char H5_ATTR_UNUSED *dest)
-{
- herr_t ret_value = FAIL;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value);
-} /* end H5FD_s3comms_HMAC_SHA256() */
-
-#endif /* H5_HAVE_ROS3_VFD */
-
/*-----------------------------------------------------------------------------
*
@@ -2621,8 +2378,6 @@ H5FD_s3comms_HMAC_SHA256(
* Programmer: Jacob Smith
* 2018-02-27
*
- * Changes: None
- *
*-----------------------------------------------------------------------------
*/
static herr_t
@@ -2781,8 +2536,6 @@ done:
* Programmer: Jacob Smith
* 2018-02-27
*
- * Changes: None
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -2903,8 +2656,6 @@ done:
* Programmer: Jacob Smith
* 2017-09-18
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -2968,8 +2719,6 @@ done:
* Programmer: Jacob Smith
* 2017-10-30
*
- * Changes: None.
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -3242,12 +2991,6 @@ done:
*
* Programmer: Jacob Smith
*
- * Changes:
- *
- * - Integrate into HDF.
- * - Rename from `hexutf8` to `H5FD_s3comms_percent_encode_char`.
- * --- Jacob Smith 2017-09-15
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -3421,19 +3164,8 @@ done:
* Programmer: Jacob Smith
* 2017-07-13
*
- * Changes:
- *
- * - Integrate into HDF5.
- * - Return herr_t type.
- * --- Jacob Smith 2017-09-18
- *
- * - NULL check and fail of input parameters.
- * --- Jacob Smith 2017-10-10
- *
*----------------------------------------------------------------------------
*/
-#ifdef H5_HAVE_ROS3_VFD
-
herr_t
H5FD_s3comms_signing_key(
unsigned char *md,
@@ -3526,22 +3258,6 @@ done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5FD_s3comms_signing_key() */
-#else /* H5_HAVE_ROS3_VFD not defined */
-
-herr_t
-H5FD_s3comms_signing_key(
- unsigned char H5_ATTR_UNUSED *md,
- const char H5_ATTR_UNUSED *secret,
- const char H5_ATTR_UNUSED *region,
- const char H5_ATTR_UNUSED *iso8601now)
-{
- herr_t ret_value = SUCCEED;
- FUNC_ENTER_NOAPI_NOINIT_NOERR;
- FUNC_LEAVE_NOAPI(ret_value);
-} /* end H5FD_s3comms_signing_key() */
-
-#endif /* H5_HAVE_ROS3_VFD */
-
/*----------------------------------------------------------------------------
*
@@ -3577,20 +3293,8 @@ H5FD_s3comms_signing_key(
* Programmer: Jacob Smith
* 2017-07-??
*
- * Changes:
- *
- * - Integrate with HDF5.
- * - Rename from `tostringtosign` to `H5FD_s3comms_tostringtosign`.
- * - Return `herr_t` instead of characters written.
- * - Use HDF-friendly bytes-to-hex function (`H5FD_s3comms_bytes_to_hex`)
- * instead of general-purpose, deprecated `hex()`.
- * - Adjust casts to openssl's `SHA256`.
- * - Input strings are now `const`.
- * --- Jacob Smith 2017-09-19
- *
*----------------------------------------------------------------------------
*/
-#ifdef H5_HAVE_ROS3_VFD
herr_t
H5FD_s3comms_tostringtosign(
char *dest,
@@ -3681,22 +3385,6 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5ros3_tostringtosign() */
-#else /* H5_HAVE_ROS3_VFD not defined */
-
-herr_t
-H5FD_s3comms_tostringtosign(
- char H5_ATTR_UNUSED *dest,
- const char H5_ATTR_UNUSED *req,
- const char H5_ATTR_UNUSED *now,
- const char H5_ATTR_UNUSED *region)
-{
- herr_t ret_value = FAIL;
- FUNC_ENTER_NOAPI_NOINIT_NOERR
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5ros3_tostringtosign() */
-
-#endif /* H5_HAVE_ROS3_VFD */
-
/*----------------------------------------------------------------------------
*
@@ -3725,13 +3413,6 @@ H5FD_s3comms_tostringtosign(
* Programmer: Jacob Smith
* 2017-09-18
*
- * Changes:
- *
- * - Rename from `trim()` to `H5FD_s3comms_trim()`.
- * - Incorporate into HDF5.
- * - Returns `herr_t` type.
- * --- Jacob Smith 2017-??-??
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -3826,16 +3507,6 @@ done:
* Programmer: Jacob Smith
* 2017-07-??
*
- * Changes:
- *
- * - Integrate to HDF environment.
- * - Rename from `uriencode` to `H5FD_s3comms_uriencode`.
- * - Change return from characters written to herr_t;
- * move to i/o parameter `n_written`.
- * - No longer append null-terminator to string;
- * programmer may append or not as appropriate upon return.
- * --- Jacob Smith 2017-09-15
- *
*----------------------------------------------------------------------------
*/
herr_t
@@ -3915,4 +3586,5 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD_s3comms_uriencode */
+#endif /* H5_HAVE_ROS3_VFD */
diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h
index 16f6a8b..94fae7e 100644
--- a/src/H5FDs3comms.h
+++ b/src/H5FDs3comms.h
@@ -52,18 +52,15 @@
*
*****************************************************************************/
-#include <ctype.h>
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
+#include "H5private.h" /* Generic Functions */
#ifdef H5_HAVE_ROS3_VFD
+
+/* Necessary S3 headers */
#include <curl/curl.h>
#include <openssl/evp.h>
#include <openssl/hmac.h>
#include <openssl/sha.h>
-#endif /* ifdef H5_HAVE_ROS3_VFD */
/*****************
* PUBLIC MACROS *
@@ -99,9 +96,6 @@
* It is left to the programmer to check return value of
* ISO8601NOW (should equal ISO8601_SIZE - 1).
*
- * Programmer: Jacob Smith
- * 2017-07-??
- *
*---------------------------------------------------------------------------
*/
#define ISO8601NOW(dest, now_gm) \
@@ -121,9 +115,6 @@ strftime((dest), ISO8601_SIZE, "%Y%m%dT%H%M%SZ", (now_gm))
* It is left to the programmer to check return value of
* RFC7231NOW (should equal RFC7231_SIZE - 1).
*
- * Programmer: Jacob Smith
- * 2017-07-??
- *
*---------------------------------------------------------------------------
*/
#define RFC7231NOW(dest, now_gm) \
@@ -169,11 +160,6 @@ strftime((dest), RFC7231_SIZE, "%a, %d %b %Y %H:%M:%S GMT", (now_gm))
* `region` should be relevant AWS region, i.e. "us-east-1".
* `service` should be "s3".
*
- * Programmer: Jacob Smith
- * 2017-09-19
- *
- * Changes: None.
- *
*---------------------------------------------------------------------------
*/
#define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service) \
@@ -266,16 +252,6 @@ HDsnprintf((dest), S3COMMS_MAX_CREDENTIAL_SIZE,
* Pointers to next node in the list, or NULL sentinel as end of list.
* Next node must have a greater `lowername` as determined by strcmp().
*
- *
- *
- * Programmer: Jacob Smith
- * 2017-09-22
- *
- * Changes:
- *
- * - Change from twin doubly-linked lists to singly-linked list.
- * --- Jake Smith 2017-01-17
- *
*----------------------------------------------------------------------------
*/
typedef struct hrb_node_t {
@@ -353,10 +329,6 @@ typedef struct hrb_node_t {
*
* Pointer to HTTP version string, e.g., "HTTP/1.1".
*
- *
- *
- * Programmer: Jacob Smith
- *
*----------------------------------------------------------------------------
*/
typedef struct {
@@ -421,10 +393,6 @@ typedef struct {
* Single string of all query parameters in url (if any).
* "arg1=value1&arg2=value2"
*
- *
- *
- * Programmer: Jacob Smith
- *
*----------------------------------------------------------------------------
*/
typedef struct {
@@ -514,15 +482,10 @@ typedef struct {
*
* Requred to authenticate.
*
- *
- *
- * Programmer: Jacob Smith
- *
*----------------------------------------------------------------------------
*/
typedef struct {
unsigned long magic;
-#ifdef H5_HAVE_ROS3_VFD
CURL *curlhandle;
size_t filesize;
char *httpverb;
@@ -530,15 +493,19 @@ typedef struct {
char *region;
char *secret_id;
unsigned char *signing_key;
-#endif /* ifdef H5_HAVE_ROS3_VFD */
} s3r_t;
+
#define S3COMMS_S3R_MAGIC 0x44d8d79
+#ifdef __cplusplus
+extern "C" {
+#endif
+
/*******************************************
* DECLARATION OF HTTP FIELD LIST ROUTINES *
*******************************************/
-herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L,
+H5_DLL herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L,
const char *name,
const char *value);
@@ -546,9 +513,9 @@ herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L,
* DECLARATION OF HTTP REQUEST BUFFER ROUTINES *
***********************************************/
-herr_t H5FD_s3comms_hrb_destroy(hrb_t **buf);
+H5_DLL herr_t H5FD_s3comms_hrb_destroy(hrb_t **buf);
-hrb_t * H5FD_s3comms_hrb_init_request(const char *verb,
+H5_DLL hrb_t * H5FD_s3comms_hrb_init_request(const char *verb,
const char *resource,
const char *host);
@@ -576,7 +543,7 @@ H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle,
H5_DLL struct tm * gmnow(void);
-herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest,
+H5_DLL herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest,
int cr_size,
char *signed_headers_dest,
int sh_size,
@@ -587,27 +554,27 @@ H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest,
size_t msg_len,
hbool_t lowercase);
-herr_t H5FD_s3comms_free_purl(parsed_url_t *purl);
+H5_DLL herr_t H5FD_s3comms_free_purl(parsed_url_t *purl);
-herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key,
+H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key,
size_t key_len,
const char *msg,
size_t msg_len,
char *dest);
-herr_t H5FD_s3comms_load_aws_profile(const char *name,
+H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name,
char *key_id_out,
char *secret_access_key_out,
char *aws_region_out);
-herr_t H5FD_s3comms_nlowercase(char *dest,
+H5_DLL herr_t H5FD_s3comms_nlowercase(char *dest,
const char *s,
size_t len);
-herr_t H5FD_s3comms_parse_url(const char *str,
+H5_DLL herr_t H5FD_s3comms_parse_url(const char *str,
parsed_url_t **purl);
-herr_t H5FD_s3comms_percent_encode_char(char *repr,
+H5_DLL herr_t H5FD_s3comms_percent_encode_char(char *repr,
const unsigned char c,
size_t *repr_len);
@@ -616,7 +583,7 @@ H5_DLL herr_t H5FD_s3comms_signing_key(unsigned char *md,
const char *region,
const char *iso8601now);
-herr_t H5FD_s3comms_tostringtosign(char *dest,
+H5_DLL herr_t H5FD_s3comms_tostringtosign(char *dest,
const char *req_str,
const char *now,
const char *region);
@@ -626,10 +593,12 @@ H5_DLL herr_t H5FD_s3comms_trim(char *dest,
size_t s_len,
size_t *n_written);
-H5_DLL herr_t H5FD_s3comms_uriencode(char *dest,
- const char *s,
- size_t s_len,
- hbool_t encode_slash,
- size_t *n_written);
+H5_DLL herr_t H5FD_s3comms_uriencode(char *dest, const char *s, size_t s_len,
+ hbool_t encode_slash, size_t *n_written);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* H5_HAVE_ROS3_VFD */
diff --git a/src/Makefile.am b/src/Makefile.am
index 1e1273c..ccee69b 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -62,9 +62,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Fsfile.c H5Fspace.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \
H5FAint.c H5FAstat.c H5FAtest.c \
- H5FD.c H5FDcore.c \
- H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c H5FDs3comms.c \
- H5FDmulti.c H5FDros3.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \
+ H5FD.c H5FDcore.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c \
+ H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \
H5FSstat.c H5FStest.c \
H5G.c H5Gbtree2.c H5Gcache.c \
@@ -134,6 +133,11 @@ if DIRECT_VFD_CONDITIONAL
libhdf5_la_SOURCES += H5FDdirect.c
endif
+# Only compile the read-only S3 VFD if necessary
+if ROS3_VFD_CONDITIONAL
+ libhdf5_la_SOURCES += H5FDros3.c H5FDs3comms.c
+endif
+
# Public headers
include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5version.h \
H5Apublic.h H5ACpublic.h \