diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | test/CMakeVFDTests.cmake | 1 | ||||
| -rw-r--r-- | test/Makefile.am | 16 | ||||
| -rw-r--r-- | test/hdfs.c | 1836 | ||||
| -rw-r--r-- | test/ros3.c | 2020 | ||||
| -rw-r--r-- | test/s3comms.c | 2813 | ||||
| -rw-r--r-- | test/vfd.c | 167 |
7 files changed, 6835 insertions, 21 deletions
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8333c78..cbd1901 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -267,6 +267,9 @@ set (H5_TESTS enc_dec_plist_cross_platform getname vfd + ros3 + s3comms + hdfs ntypes dangle dtransform diff --git a/test/CMakeVFDTests.cmake b/test/CMakeVFDTests.cmake index 2767bdc..7f661a5 100644 --- a/test/CMakeVFDTests.cmake +++ b/test/CMakeVFDTests.cmake @@ -18,6 +18,7 @@ # included from CMakeTEsts.cmake set (VFD_LIST + hdfs sec2 stdio core diff --git a/test/Makefile.am b/test/Makefile.am index 1b82aa4..dbee9c2 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -59,11 +59,11 @@ TEST_PROG= testhdf5 \ stab gheap evict_on_close farray earray btree2 fheap \ pool accum hyperslab istore bittests dt_arith page_buffer \ dtypes dsets cmpd_dset filter_fail extend direct_chunk external efc \ - objcopy links unlink twriteorder big mtime fillval mount \ - flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ - enc_dec_plist_cross_platform getname vfd ntypes dangle dtransform \ - reserved cross_read freespace mf vds file_image unregister \ - cache_logging cork swmr vol + objcopy links unlink twriteorder big mtime fillval mount flush1 \ + flush2 app_ref enum set_extent ttsafe enc_dec_plist \ + enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ + dangle dtransform reserved cross_read freespace mf vds file_image \ + unregister cache_logging cork swmr vol # List programs to be built when testing here. # error_test and err_compat are built at the same time as the other tests, but executed by testerror.sh. @@ -145,7 +145,7 @@ ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \ ttsafe_acreate.c cache_image_SOURCES=cache_image.c genall5.c -VFD_LIST = sec2 stdio core core_paged split multi family +VFD_LIST = hdfs sec2 stdio core core_paged split multi family if DIRECT_VFD_CONDITIONAL VFD_LIST += direct endif @@ -201,8 +201,8 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse dtransform.h5 test_filters.h5 get_file_name.h5 tstint[1-2].h5 \ unlink_chunked.h5 btree2.h5 btree2_tmp.h5 objcopy_src.h5 objcopy_dst.h5 \ objcopy_ext.dat trefer1.h5 trefer2.h5 app_ref.h5 farray.h5 farray_tmp.h5 \ - earray.h5 earray_tmp.h5 efc[0-5].h5 log_vfd_out.log \ - new_multi_file_v16-r.h5 new_multi_file_v16-s.h5 \ + earray.h5 earray_tmp.h5 efc[0-5].h5 log_vfd_out.log log_ros3_out.log \ + log_s3comms_out.log new_multi_file_v16-r.h5 new_multi_file_v16-s.h5 \ split_get_file_image_test-m.h5 split_get_file_image_test-r.h5 \ file_image_core_test.h5.copy unregister_filter_1.h5 unregister_filter_2.h5 \ vds_virt.h5 vds_dapl.h5 vds_src_[0-1].h5 \ diff --git a/test/hdfs.c b/test/hdfs.c new file mode 100644 index 0000000..57dbac9 --- /dev/null +++ b/test/hdfs.c @@ -0,0 +1,1836 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only HDFS Virtual File Driver (VFD) + * + * Purpose: + * + * Verify behavior for Read-Only HDFS VFD. + * + * Demonstrates basic use cases and fapl interaction. + * + * Programmer: Jacob Smith <jake.smith@hdfgroup.org> + * 2018-04-23 + */ + +#include "h5test.h" /* testing utilities */ +#include "H5FDhdfs.h" /* this file driver's utilities */ + + +#define HDFS_TEST_DEBUG 0 +#define HDFS_TEST_MAX_BUF_SIZE 256 +#if 0 +#define S3_TEST_MAX_URL_SIZE 256 +#endif /* s3comms relic */ + +/***************************************************************************** + * + * FILE-LOCAL TESTING MACROS + * + * Purpose: + * + * 1) Upon test failure, goto-jump to single-location teardown in test + * function. E.g., `error:` (consistency with HDF corpus) or + * `failed:` (reflects purpose). + * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. + * 2) Increase clarity and reduce overhead found with `TEST_ERROR`. + * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:" + * requires reading of entire line to know whether this if/call is + * part of the test setup, test operation, or a test unto itself. + * 3) Provide testing macros with optional user-supplied failure message; + * if not supplied (NULL), generate comparison output in the spirit of + * test-driven development. E.g., "expected 5 but was -3" + * User messages clarify test's purpose in code, encouraging description + * without relying on comments. + * 4) Configurable expected-actual order in generated comparison strings. + * Some prefer `VERIFY(expected, actual)`, others + * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch + * to satifsy both parties, assuming one paradigm per test file. + * (One could #undef and redefine the flag through the file as desired, + * but _why_.) + * + * Provided as courtesy, per consideration for inclusion in the library + * proper. + * + * Macros: + * + * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order + * FAIL_IF() - check condition + * FAIL_UNLESS() - check _not_ condition + * JSVERIFY() - long-int equality check; prints reason/comparison + * JSVERIFY_NOT() - long-int inequality check; prints + * JSVERIFY_STR() - string equality check; prints + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * ifdef flag: JSVERIFY_EXP_ACT + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_EXP_ACT 1L + + +/*---------------------------------------------------------------------------- + * + * Macro: JSFAILED_AT() + * + * Purpose: + * + * Preface a test failure by printing "*FAILED*" and location to stdout + * Similar to `H5_FAILED(); AT();` from h5test.h + * + * *FAILED* at somefile.c:12 in function_name()... + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSFAILED_AT() { \ + HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_IF() + * + * Purpose: + * + * Make tests more accessible and less cluttered than + * `if (thing == otherthing()) TEST_ERROR` + * paradigm. + * + * The following lines are roughly equivalent: + * + * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests) + * `FAIL_IF(myfunc() < 0)` + * + * Prints a generic "FAILED AT" line to stdout and jumps to `error`, + * similar to `TEST_ERROR` in h5test.h + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_IF(condition) \ +if (condition) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_UNLESS() + * + * Purpose: + * + * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", + * e.g., "a != b". + * + * Opposite of FAIL_IF; fails if the given condition is _not_ true. + * + * `FAIL_IF( 5 != my_op() )` + * is equivalent to + * `FAIL_UNLESS( 5 == my_op() )` + * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. + * (see JSVERIFY) + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#if 0 /* UNUSED */ +#define FAIL_UNLESS(condition) \ +if (!(condition)) { \ + JSFAILED_AT() \ + goto error; \ +} +#endif /* UNUSED */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_LONG() + * + * Purpose: + * + * Print an failure message for long-int arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:488 in somefunc()... + * forest must be made of trees. + * + * or + * + * *FAILED* at myfile.c:488 in somefunc()... + * ! Expected 425 + * ! Actual 3 + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_LONG(expected, actual, reason) { \ + JSFAILED_AT() \ + if (reason!= NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf(" ! Expected %ld\n ! Actual %ld\n", \ + (long)(expected), (long)(actual)); \ + } \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_STR() + * + * Purpose: + * + * Print an failure message for string arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:421 in myfunc()... + * Blue and Red strings don't match! + * + * or + * + * *FAILED* at myfile.c:421 in myfunc()... + * !!! Expected: + * this is my expected + * string + * !!! Actual: + * not what I expected at all + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_STR(expected, actual, reason) { \ + JSFAILED_AT() \ + if ((reason) != NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \ + (expected), (actual)); \ + } \ +} + + + +#ifdef JSVERIFY_EXP_ACT + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY() + * + * Purpose: + * + * Verify that two long integers are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(expected, actual, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_NOT() + * + * Purpose: + * + * Verify that two long integers are _not_ equal. + * If equal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(expected, actual, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_STR() + * + * Purpose: + * + * Verify that two strings are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(expected, actual, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + + +#else +/* JSVERIFY_EXP_ACT not defined + * + * Repeats macros above, but with actual/expected parameters reversed. + */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY() + * See: JSVERIFY documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(actual, expected, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_NOT() + * See: JSVERIFY_NOT documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(actual, expected, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_STR() + * See: JSVERIFY_STR documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(actual, expected, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + +#endif /* ifdef/else JSVERIFY_EXP_ACT */ + +/******************************** + * OTHER MACROS AND DEFINITIONS * + ********************************/ + +/* copied from src/hdfs.c + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define MAX_HDFS_NAMENODE_NAME 128 + +/******************************* + * FILE-LOCAL GLOBAL VARIABLES * + *******************************/ +static const char filename_missing[] = "/tmp/missing.txt"; +static const char filename_bard[] = "/tmp/t8.shakespeare.txt"; +static const char filename_raven[] = "/tmp/Poe_Raven.txt"; +static const char filename_example_h5[] = "/tmp/t.h5"; +static H5FD_hdfs_fapl_t default_fa = { + 1, /* fa version */ + "localhost", /* namenode name */ + 0, /* namenode port */ + "", /* user name */ + "", /* kerberos path */ + 1024, /* buffer size */ +}; + +/****************** + * TEST FUNCTIONS * + ******************/ + + +/*--------------------------------------------------------------------------- + * + * Function: test_fapl_config_validation() + * + * Purpose: + * + * Test data consistency of fapl configuration. + * Tests `H5FD_hdfs_validate_config` indirectly through `H5Pset_fapl_hdfs`. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-04-25 + * + * Changes: None. + * + *--------------------------------------------------------------------------- + */ +static int +test_fapl_config_validation(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *msg; + herr_t expected; + H5FD_hdfs_fapl_t config; + }; + + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; /* file access property list ID */ + H5FD_hdfs_fapl_t config; + H5FD_hdfs_fapl_t fa_fetch; + herr_t success = SUCCEED; + unsigned int i = 0; + unsigned int ncases = 6; /* should equal number of cases */ + struct testcase *case_ptr = NULL; /* dumb work-around for possible */ + /* dynamic cases creation because */ + /* of compiler warnings Wlarger-than */ + struct testcase cases_arr[] = { + { "default config fapl", + SUCCEED, + { 1, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "invalid version number (2)", + FAIL, + { 2, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "invalid version number (0)", + FAIL, + { 0, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "nonsense kerberos path still ok?", + SUCCEED, + { 1, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "pathToSomewhere", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "namenode port number too high", + FAIL, + { 1, /* version */ + "localhost", /* namenode_name */ + 88000, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "negative namenode port number", + FAIL, + { 1, /* version */ + "localhost", /* namenode_name */ + -1, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + }; + + TESTING("HDFS fapl configuration validation"); + + /********* + * TESTS * + *********/ + + for (i = 0; i < ncases; i++) { + + /*--------------- + * per-test setup + *--------------- + */ + case_ptr = &cases_arr[i]; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) /* sanity-check */ + + /*----------------------------------- + * Actually test -- set fapl. + * Mute stack trace in failure cases. + *----------------------------------- + */ + H5E_BEGIN_TRY { + /* `H5FD_hdfs_validate_config(...)` is static/private + * to src/hdfs.c and cannot (and should not?) be tested directly? + * Instead, validate config through public api. + */ + success = H5Pset_fapl_hdfs(fapl_id, &case_ptr->config); + } H5E_END_TRY; + + JSVERIFY( case_ptr->expected, success, case_ptr->msg ) + + /* Make sure we can get back what we put in. + * Only valid if the fapl configuration does not result in error. + */ + if (success == SUCCEED) { + config = case_ptr->config; + JSVERIFY( SUCCEED, + H5Pget_fapl_hdfs(fapl_id, &fa_fetch), + "unable to get fapl" ) + JSVERIFY( H5FD__CURR_HDFS_FAPL_T_VERSION, + fa_fetch.version, + "invalid version number" ) + JSVERIFY( config.version, + fa_fetch.version, + "version number mismatch" ) + JSVERIFY( config.namenode_port, + fa_fetch.namenode_port, + "namenode port mismatch" ) + JSVERIFY( config.stream_buffer_size, + fa_fetch.stream_buffer_size, + "streambuffer size mismatch" ) + JSVERIFY_STR( config.namenode_name, + fa_fetch.namenode_name, + NULL ) + JSVERIFY_STR( config.user_name, + fa_fetch.user_name, + NULL ) + JSVERIFY_STR( config.kerberos_ticket_cache, + fa_fetch.kerberos_ticket_cache, + NULL ) + } + + /*----------------------------- + * per-test sanitation/teardown + *----------------------------- + */ + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + } /* for each test case */ + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fapl_id < 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + return 1; + +} /* test_fapl_config_validation */ + + +/*------------------------------------------------------------------------- + * + * Function: test_hdfs_fapl() + * + * Purpose: Tests the file handle interface for the HDFS driver. + * + * For now, test only fapl & flags. Extend as the + * work on the VFD continues. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2018-04-25 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static int +test_hdfs_fapl(void) +{ + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; /* file access property list ID */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + H5FD_hdfs_fapl_t hdfs_fa_0 = { + 1, /* version*/ + "", /* node name */ + 9000, /* node port */ + "", /* username */ + "", /* kerb cache path */ + 1024, /* stream buffer size */ + }; + + TESTING("HDFS fapl "); + + /* Set property list and file name for HDFS driver. + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &hdfs_fa_0) ) + + driver_id = H5Pget_driver(fapl_id); + FAIL_IF( driver_id < 0 ) + + /**************** + * Check that the VFD feature flags are correct + * SPEC MAY CHANGE + ******************/ + + FAIL_IF( H5FDdriver_query(driver_id, &driver_flags) < 0 ) + + JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE), + "bit(s) in `driver_flags` must align with " + "H5FD_FEAT_DATA_SIEVE" ) + + JSVERIFY( H5FD_FEAT_DATA_SIEVE, driver_flags, + "H5FD_FEAT_DATA_SIEVE should be the only supported flag") + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + + return 1; + +} /* test_hdfs_fapl() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_vfd_open() + * + * Purpose: + * + * Demonstrate/specify VFD-level "Open" failure cases + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-06-07 + * + *--------------------------------------------------------------------------- + */ +static int +test_vfd_open(void) +{ + /********************* + * test-local macros * + *********************/ + +/* selectors for which fapl to use in testcase */ +#define FAPL_H5P_DEFAULT -2 +#define FAPL_UNCONFIGURED -3 /* H5P_FILE_ACCESS */ +#define FAPL_HDFS -4 + +#ifdef H5_HAVE_LIBHDFS + /************************* + * test-local structures * + *************************/ + + struct test_condition { + const char *message; + const char *url; + unsigned flags; + int which_fapl; + haddr_t maxaddr; + hbool_t might_use_other_driver; + }; + + /************************ + * test-local variables * + ************************/ + + struct test_condition failing_conditions[] = { + { "default property list (H5P_DEFAULT) is invalid", + filename_bard, + H5F_ACC_RDONLY, + FAPL_H5P_DEFAULT, + MAXADDR, + TRUE, + }, + { "generic file access property list is invalid", + filename_bard, + H5F_ACC_RDONLY, + FAPL_UNCONFIGURED, + MAXADDR, + TRUE, + }, + { "filename cannot be null", + NULL, + H5F_ACC_RDONLY, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "filename cannot be empty", + "", + H5F_ACC_RDONLY, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "file at filename must exist", + filename_missing, + H5F_ACC_RDONLY, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "read-write flag not supported", + filename_bard, + H5F_ACC_RDWR, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "truncate flag not supported", + filename_bard, + H5F_ACC_TRUNC, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "create flag not supported", + filename_bard, + H5F_ACC_CREAT, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "EXCL flag not supported", + filename_bard, + H5F_ACC_EXCL, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "maxaddr cannot be 0 (caught in `H5FD_open()`)", + filename_bard, + H5F_ACC_RDONLY, + FAPL_HDFS, + 0, + FALSE, + }, + }; +#endif /* H5_HAVE_LIBHDFS */ + H5FD_t *fd = NULL; + hid_t fapl_hdfs = -1; + hid_t fapl_unconfigured = -1; + unsigned i = 0; + unsigned failing_conditions_count = 10; + + TESTING("HDFS VFD-level open"); + +#ifndef H5_HAVE_LIBHDFS + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; +#else + fapl_unconfigured = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_unconfigured < 0 ) + + fapl_hdfs = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_hdfs < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_hdfs, &default_fa) ) + + /********* + * TESTS * + *********/ + + /* all the test cases that will _not_ open + */ + for (i = 0; i < failing_conditions_count; i++) { + struct test_condition T = failing_conditions[i]; + hid_t fapl_id = H5P_DEFAULT; + + fd = NULL; + + if (T.which_fapl == FAPL_UNCONFIGURED) + fapl_id = fapl_unconfigured; + else if (T.which_fapl == FAPL_HDFS) + fapl_id = fapl_hdfs; + +#if HDFS_TEST_DEBUG + HDfprintf(stderr, "testing: %s\n", T.message); +#endif /* HDFS_TEST_DEBUG */ + + H5E_BEGIN_TRY { + fd = H5FDopen(T.url, T.flags, fapl_id, T.maxaddr); + } H5E_END_TRY; + if (NULL != fd) { + if (TRUE == T.might_use_other_driver && + H5FD_HDFS != fd->driver_id) + { + HDfprintf(stderr, "\n!!!!! WARNING !!!!!\n" \ + " Successful open of file on local system " \ + "with non-HDFS VFD.\n"); + JSVERIFY(SUCCEED, H5FDclose(fd), + "unable to close errant open"); + fd = NULL; + } else { + JSVERIFY(1, 0, T.message); /* print message and fail */ + } + } + } + + FAIL_IF( NULL != fd ) /* sanity check */ + +#if HDFS_TEST_DEBUG + HDfprintf(stderr, "nominal open\n"); +#endif /* HDFS_TEST_DEBUG */ + + /* finally, show that a file can be opened + */ + fd = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_hdfs, + MAXADDR); + FAIL_IF( NULL == fd ) + + /************ + * TEARDOWN * + ************/ + +#if HDFS_TEST_DEBUG + HDfprintf(stderr, "teardown...\n"); +#endif /* HDFS_TEST_DEBUG */ + + FAIL_IF( FAIL == H5FDclose(fd) ) + fd = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_hdfs) ) + fapl_hdfs = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_unconfigured) ) + fapl_unconfigured = -1; + + PASSED(); + return 0; +#endif /* H5_HAVE_LIBHDFS */ + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd) + (void)H5FDclose(fd); + H5E_BEGIN_TRY { + if (fapl_hdfs >= 0) + (void)H5Pclose(fapl_hdfs); + if (fapl_unconfigured >= 0) + (void)H5Pclose(fapl_unconfigured); + } H5E_END_TRY; + + return 1; + +#undef FAPL_H5P_DEFAULT +#undef FAPL_UNCONFIGURED +#undef FAPL_HDFS + +} /* test_vfd_open */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_eof_eoa() + * + * Purpose: + * + * Demonstrate behavior of get_eof, get_eoa, and set_eoa. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-06-07 + * + *--------------------------------------------------------------------------- + */ +static int +test_eof_eoa(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + H5FD_t *fd_shakespeare = NULL; + hid_t fapl_id = -1; + + TESTING("HDFS eof/eoa gets and sets"); + +#ifndef H5_HAVE_LIBHDFS + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; +#else + /********* + * SETUP * + *********/ + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + fd_shakespeare = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_shakespeare ) + + /********* + * TESTS * + *********/ + + /* verify as found + */ + JSVERIFY( 5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), NULL ) + JSVERIFY( H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW), + "mismatch between DEFAULT and RAW memory types" ) + JSVERIFY( 0, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA should be unset by H5FDopen" ) + + /* set EoA below EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202), + "unable to set EoA (lower)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 44442202, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /* set EoA above EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012), + "unable to set EoA (higher)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 6789012, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd_shakespeare) ) + fd_shakespeare = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; +#endif /* H5_HAVE_LIBHDFS */ + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd_shakespeare != NULL) { + (void)H5FDclose(fd_shakespeare); + } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +} /* test_eof_eoa */ + + +/*----------------------------------------------------------------------------- + * + * Function: test_H5FDread_without_eoa_set_fails() + * + * Purpose: + * + * Demonstrate a not-obvious constraint by the library, preventing + * file read before EoA is set + * + * Programmer: Jacob Smith + * 2018-06-08 + * + *----------------------------------------------------------------------------- + */ +static int +test_H5FDread_without_eoa_set_fails(void) +{ +#ifdef H5_HAVE_LIBHDFS + char buffer[HDFS_TEST_MAX_BUF_SIZE]; + unsigned int i = 0; +#endif /* H5_HAVE_LIBHDFS */ + H5FD_t *file_shakespeare = NULL; + hid_t fapl_id = -1; + + TESTING("HDFS VFD read-eoa temporal coupling library limitation"); + +#ifndef H5_HAVE_LIBHDFS + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; +#else + /********* + * SETUP * + *********/ + + /* create HDFS fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + file_shakespeare = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_id, + MAXADDR); + FAIL_IF( NULL == file_shakespeare ) + + JSVERIFY( 0, H5FDget_eoa(file_shakespeare, H5FD_MEM_DEFAULT), + "EoA should remain unset by H5FDopen" ) + + /* zero buffer contents */ + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + buffer[i] = 0; + } + + /******** + * TEST * + ********/ + + H5E_BEGIN_TRY { /* mute stack trace on expected failure */ + JSVERIFY( FAIL, + H5FDread(file_shakespeare, + H5FD_MEM_DRAW, + H5P_DEFAULT, + 1200699, + 102, + buffer), + "cannot read before eoa is set" ) + } H5E_END_TRY; + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + JSVERIFY( 0, (unsigned)buffer[i], "buffer was modified by write!" ) + } + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_shakespeare) ) + file_shakespeare = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; +#endif /* H5_HAVE_LIBHDFS */ + +error: + /*********** + * CLEANUP * + ***********/ + + if (file_shakespeare) { + (void)H5FDclose(file_shakespeare); + } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +} /* test_H5FDread_without_eoa_set_fails */ + + + +/*--------------------------------------------------------------------------- + * + * Function: test_read() + * + * Purpose: + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-06-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_read(void) +{ + /********************* + * test-local macros * + *********************/ + +#ifdef H5_HAVE_LIBHDFS + /************************* + * test-local structures * + *************************/ + struct testcase { + const char *message; /* purpose of test case */ + haddr_t eoa_set; /* set file EOA to this prior to read */ + size_t addr; /* offset of read in file */ + size_t len; /* length of read in file */ + herr_t success; /* expected return value of read function */ + const char *expected; /* expected contents of buffer; failure ignores */ + }; + + /************************ + * test-local variables * + ************************/ + struct testcase cases[] = { + { "successful range-get", + 6464, + 5691, + 32, /* fancy quotes are three bytes each(?) */ + SUCCEED, + "Quoth the Raven “Nevermore.”", + }, + { "read past EOA fails (EOA < EOF < addr)", + 3000, + 4000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA fails (EOA < addr < EOF < (addr+len))", + 3000, + 8000, + 100, + FAIL, + NULL, + }, + { "read past EOA/EOF fails ((EOA==EOF) < addr)", + 6464, + 7000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA/EOF fails (addr < (EOA==EOF) < (addr+len))", + 6464, + 6400, + 100, + FAIL, + NULL, + }, + { "read between EOF and EOA fails (EOF < addr < (addr+len) < EOA)", + 8000, + 7000, + 100, + FAIL, + NULL, + }, + }; + unsigned testcase_count = 6; + unsigned test_i = 0; + struct testcase test; + herr_t open_return = FAIL; + char buffer[HDFS_TEST_MAX_BUF_SIZE]; + unsigned int i = 0; +#endif /* H5_HAVE_LIBHDFS */ + H5FD_t *file_raven = NULL; + hid_t fapl_id = -1; + + TESTING("HDFS VFD read/range-gets"); + +#ifndef H5_HAVE_LIBHDFS + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; +#else + /********* + * SETUP * + *********/ + + /* create HDFS fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + /* zero buffer contents */ + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + buffer[i] = 0; + } + + /* open file + */ + file_raven = H5FDopen( + filename_raven, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); /* Demonstrate success with "automatic" value */ + FAIL_IF( NULL == file_raven ) + + JSVERIFY( 6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), NULL ) + + /********* + * TESTS * + *********/ + + for (test_i = 0; test_i < testcase_count; test_i++) { + + /* -------------- * + * per-test setup * + * -------------- */ + + test = cases[test_i]; + open_return = FAIL; + + FAIL_IF( HDFS_TEST_MAX_BUF_SIZE < test.len ) /* buffer too small! */ + + FAIL_IF( FAIL == + H5FDset_eoa( file_raven, H5FD_MEM_DEFAULT, test.eoa_set) ) + + /* zero buffer contents */ + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + buffer[i] = 0; + } + + /* ------------ * + * conduct test * + * ------------ */ + + H5E_BEGIN_TRY { + open_return = H5FDread( + file_raven, + H5FD_MEM_DRAW, + H5P_DEFAULT, + test.addr, + test.len, + buffer); + } H5E_END_TRY; + + JSVERIFY( test.success, + open_return, + test.message ) + + if (open_return == SUCCEED) { + JSVERIFY_STR( test.expected, buffer, NULL ) + } + + } /* for each testcase */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_raven) ) + file_raven = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; +#endif /* H5_HAVE_LIBHDFS */ + +error: + /*********** + * CLEANUP * + ***********/ + + if (file_raven != 0) + (void)H5FDclose(file_raven); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +} /* test_read */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_noops_and_autofails() + * + * Purpose: + * + * Demonstrate the unavailable and do-nothing routines unique to + * Read-Only VFD. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + * Changes: + * + modify from S3VFD codebase to HDFS; Minor changes, mostly. + * + Jacob Smith 2018-06-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_noops_and_autofails(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; + H5FD_t *file = NULL; + const char data[36] = "The Force shall be with you, always"; + + TESTING("HDFS VFD always-fail and no-op routines"); + + /********* + * SETUP * + *********/ + +#ifndef H5_HAVE_LIBHDFS + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; +#else + /* create HDFS fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + /* open file + */ + file = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == file ) + + /********* + * TESTS * + *********/ + + /* auto-fail calls to write and truncate + */ + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data), + "write must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, FALSE), + "truncate must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, TRUE), + "truncate must fail (closing)" ) + } H5E_END_TRY; + + /* no-op calls to `lock()` and `unlock()` + */ + JSVERIFY( SUCCEED, + H5FDlock(file, TRUE), + "lock always succeeds; has no effect" ) + JSVERIFY( SUCCEED, + H5FDlock(file, FALSE), + NULL ) + JSVERIFY( SUCCEED, + H5FDunlock(file), + NULL ) + /* Lock/unlock with null file or similar error crashes tests. + * HDassert in calling heirarchy, `H5FD[un]lock()` and `H5FD_[un]lock()` + */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file) ) + file = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; +#endif /* H5_HAVE_LIBHDFS */ + +error: + /*********** + * CLEANUP * + ***********/ + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file != NULL) { + (void)H5FDclose(file); + } + + return 1; + +} /* test_noops_and_autofails*/ + + +/*--------------------------------------------------------------------------- + * + * Function: test_cmp() + * + * Purpose: + * + * Verify "file comparison" behavior. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_cmp(void) +{ +#if 0 + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + H5FD_t *fd_raven = NULL; + H5FD_t *fd_shakes = NULL; + H5FD_t *fd_raven_2 = NULL; + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; + + + + TESTING("HDFS cmp (comparison)"); + + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + JSVERIFY( SUCCEED, H5Pset_fapl_hdfs(fapl_id, &restricted_access_fa), NULL ) + + fd_raven = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_raven ) + + fd_shakes = H5FDopen( + url_text_restricted, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_shakes ) + + fd_raven_2 = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_raven_2 ) + + /********* + * TESTS * + *********/ + + JSVERIFY( 0, H5FDcmp(fd_raven, fd_raven_2), NULL ) + JSVERIFY( -1, H5FDcmp(fd_raven, fd_shakes), NULL ) + JSVERIFY( 1, H5FDcmp(fd_shakes, fd_raven_2), NULL ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd_raven) ) + fd_raven = NULL; + FAIL_IF( FAIL == H5FDclose(fd_shakes) ) + fd_shakes = NULL; + FAIL_IF( FAIL == H5FDclose(fd_raven_2) ) + fd_raven_2 = NULL; + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd_raven != NULL) (void)H5FDclose(fd_raven); + if (fd_raven_2 != NULL) (void)H5FDclose(fd_raven_2); + if (fd_shakes != NULL) (void)H5FDclose(fd_shakes); + if (TRUE == curl_ready) curl_global_cleanup(); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + +#else + /* + JSFAILED_AT() + HDprintf("TODO\n"); + return 1; + */ + + TESTING("HDFS cmp (comparison)"); + SKIPPED(); + HDfprintf( + stderr, + " TODO: Distinct valid fapls to open the same file.\n"); + + return 0; +#endif /* s3comms relic */ + +} /* test_cmp */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_H5F_integration() + * + * Purpose: + * + * Demonstrate H5F (File interface) behavior with files on HDFS. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-07 + * + * Changes: + * + modify from S3VFD codebase to HDFS; Minor changes, mostly. + * + Jacob Smith 2018-06-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_H5F_integration(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + hid_t file = -1; + hid_t fapl_id = -1; + + TESTING("HDFS file access through HD5F library (H5F API)"); + + /********* + * SETUP * + *********/ + +#ifndef H5_HAVE_LIBHDFS + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; +#else + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + /********* + * TESTS * + *********/ + + /* Read-Write Open access is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fopen( + filename_example_h5, + H5F_ACC_RDWR, + fapl_id) ) + } H5E_END_TRY; + + /* H5Fcreate() is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fcreate( + filename_missing, + H5F_ACC_RDONLY, + H5P_DEFAULT, + fapl_id) ) + } H5E_END_TRY; + + /* Successful open. + */ + file = H5Fopen( + filename_example_h5, + H5F_ACC_RDONLY, + fapl_id); + FAIL_IF( file < 0 ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5Fclose(file) ) + file = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; +#endif /* H5_HAVE_LIBHDFS */ + +error: + /*********** + * CLEANUP * + ***********/ + +#if HDFS_TEST_DEBUG + HDprintf("\nerror!"); fflush(stdout); +#endif /* HDFS_TEST_DEBUG */ + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file > 0) { + (void)H5Fclose(file); + } + + return 1; + +} /* test_H5F_integration */ + + +/*------------------------------------------------------------------------- + * + * Function: main + * + * Purpose: Tests the basic features of Virtual File Drivers + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + + /****************** + * commence tests * + ******************/ + + static char hdfs_namenode_name[MAX_HDFS_NAMENODE_NAME] = ""; + const char *hdfs_namenode_name_env = NULL; + + hdfs_namenode_name_env = HDgetenv("HDFS_TEST_NAMENODE_NAME"); + if (hdfs_namenode_name_env == NULL || hdfs_namenode_name_env[0] == '\0') { + HDstrncpy(hdfs_namenode_name, "localhost", 9); + } else { + HDstrncpy(default_fa.namenode_name, hdfs_namenode_name_env, MAX_HDFS_NAMENODE_NAME); + } + + h5_reset(); + + HDprintf("Testing hdfs VFD functionality.\n"); + + nerrors += test_fapl_config_validation(); + nerrors += test_hdfs_fapl(); + nerrors += test_vfd_open(); + nerrors += test_eof_eoa(); + nerrors += test_H5FDread_without_eoa_set_fails(); + nerrors += test_read(); + nerrors += test_noops_and_autofails(); + nerrors += test_cmp(); + nerrors += test_H5F_integration(); + + if (nerrors > 0) { + HDprintf("***** %d hdfs TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + nerrors = 1; + } else { + HDprintf("All hdfs tests passed.\n"); + } + return nerrors; /* 0 if no errors, 1 if any errors */ + +} /* main() */ + + diff --git a/test/ros3.c b/test/ros3.c new file mode 100644 index 0000000..60d3932 --- /dev/null +++ b/test/ros3.c @@ -0,0 +1,2020 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Purpose: + * + * Verify behavior for Read-Only S3 VFD + * at the VFL (virtual file layer) level. + * + * Demonstrates basic use cases and fapl/dxpl interaction. + * + * Programmer: Jacob Smith <jake.smith@hdfgroup.org> + * 2017-10-11 + */ + +#include "h5test.h" + +#include "H5FDprivate.h" /* Virtual File Driver utilities */ +#include "H5FDros3.h" /* this file driver's utilities */ +#include "H5FDs3comms.h" /* for loading of credentials */ + + + +/***************************************************************************** + * + * FILE-LOCAL TESTING MACROS + * + * Purpose: + * + * 1) Upon test failure, goto-jump to single-location teardown in test + * function. E.g., `error:` (consistency with HDF corpus) or + * `failed:` (reflects purpose). + * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. + * 2) Increase clarity and reduce overhead found with `TEST_ERROR`. + * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:" + * requires reading of entire line to know whether this if/call is + * part of the test setup, test operation, or a test unto itself. + * 3) Provide testing macros with optional user-supplied failure message; + * if not supplied (NULL), generate comparison output in the spirit of + * test-driven development. E.g., "expected 5 but was -3" + * User messages clarify test's purpose in code, encouraging description + * without relying on comments. + * 4) Configurable expected-actual order in generated comparison strings. + * Some prefer `VERIFY(expected, actual)`, others + * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch + * to satifsy both parties, assuming one paradigm per test file. + * (One could #undef and redefine the flag through the file as desired, + * but _why_.) + * + * Provided as courtesy, per consideration for inclusion in the library + * proper. + * + * Macros: + * + * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order + * FAIL_IF() - check condition + * FAIL_UNLESS() - check _not_ condition + * JSVERIFY() - long-int equality check; prints reason/comparison + * JSVERIFY_NOT() - long-int inequality check; prints + * JSVERIFY_STR() - string equality check; prints + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * ifdef flag: JSVERIFY_EXP_ACT + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_EXP_ACT 1L + + +/*---------------------------------------------------------------------------- + * + * Macro: JSFAILED_AT() + * + * Purpose: + * + * Preface a test failure by printing "*FAILED*" and location to stdout + * Similar to `H5_FAILED(); AT();` from h5test.h + * + * *FAILED* at somefile.c:12 in function_name()... + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSFAILED_AT() { \ + HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_IF() + * + * Purpose: + * + * Make tests more accessible and less cluttered than + * `if (thing == otherthing()) TEST_ERROR` + * paradigm. + * + * The following lines are roughly equivalent: + * + * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests) + * `FAIL_IF(myfunc() < 0)` + * + * Prints a generic "FAILED AT" line to stdout and jumps to `error`, + * similar to `TEST_ERROR` in h5test.h + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_IF(condition) \ +if (condition) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_UNLESS() + * + * Purpose: + * + * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", + * e.g., "a != b". + * + * Opposite of FAIL_IF; fails if the given condition is _not_ true. + * + * `FAIL_IF( 5 != my_op() )` + * is equivalent to + * `FAIL_UNLESS( 5 == my_op() )` + * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. + * (see JSVERIFY) + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#if 0 /* UNUSED */ +#define FAIL_UNLESS(condition) \ +if (!(condition)) { \ + JSFAILED_AT() \ + goto error; \ +} +#endif + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_LONG() + * + * Purpose: + * + * Print an failure message for long-int arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:488 in somefunc()... + * forest must be made of trees. + * + * or + * + * *FAILED* at myfile.c:488 in somefunc()... + * ! Expected 425 + * ! Actual 3 + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_LONG(expected, actual, reason) { \ + JSFAILED_AT() \ + if (reason!= NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf(" ! Expected %ld\n ! Actual %ld\n", \ + (long)(expected), (long)(actual)); \ + } \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_STR() + * + * Purpose: + * + * Print an failure message for string arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:421 in myfunc()... + * Blue and Red strings don't match! + * + * or + * + * *FAILED* at myfile.c:421 in myfunc()... + * !!! Expected: + * this is my expected + * string + * !!! Actual: + * not what I expected at all + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_STR(expected, actual, reason) { \ + JSFAILED_AT() \ + if ((reason) != NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \ + (expected), (actual)); \ + } \ +} + + + +#ifdef JSVERIFY_EXP_ACT + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY() + * + * Purpose: + * + * Verify that two long integers are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(expected, actual, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_NOT() + * + * Purpose: + * + * Verify that two long integers are _not_ equal. + * If equal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(expected, actual, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_STR() + * + * Purpose: + * + * Verify that two strings are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(expected, actual, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + + +#else +/* JSVERIFY_EXP_ACT not defined + * + * Repeats macros above, but with actual/expected parameters reversed. + */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY() + * See: JSVERIFY documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(actual, expected, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_NOT() + * See: JSVERIFY_NOT documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(actual, expected, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_STR() + * See: JSVERIFY_STR documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(actual, expected, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + +#endif /* ifdef/else JSVERIFY_EXP_ACT */ + +/******************************** + * OTHER MACROS AND DEFINITIONS * + ********************************/ + +/* copied from src/ros3.c + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) + +#ifdef H5_HAVE_ROS3_VFD +#define S3_TEST_PROFILE_NAME "ros3_vfd_test" + +#define S3_TEST_MAX_URL_SIZE 256 + +#define S3_TEST_RESOURCE_TEXT_RESTRICTED "t8.shakespeare.txt" +#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt" +#define S3_TEST_RESOURCE_H5_PUBLIC "GMODO-SVM01.h5" +#define S3_TEST_RESOURCE_MISSING "missing.csv" + +static char url_text_restricted[S3_TEST_MAX_URL_SIZE] = ""; +static char url_text_public[S3_TEST_MAX_URL_SIZE] = ""; +static char url_h5_public[S3_TEST_MAX_URL_SIZE] = ""; +static char url_missing[S3_TEST_MAX_URL_SIZE] = ""; +static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; +static hbool_t s3_test_bucket_defined = FALSE; + +/* Global variables for aws test profile. + * An attempt is made to read ~/.aws/credentials and ~/.aws/config upon test + * startup -- if unable to open either file or cannot load region, id, and key, + * tests connecting with S3 will not be run + */ +static int s3_test_credentials_loaded = 0; +static char s3_test_aws_region[16]; +static char s3_test_aws_access_key_id[64]; +static char s3_test_aws_secret_access_key[128]; + +H5FD_ros3_fapl_t restricted_access_fa = { + H5FD__CURR_ROS3_FAPL_T_VERSION, /* fapl version */ + TRUE, /* authenticate */ + "", /* aws region */ + "", /* access key id */ + ""}; /* secret access key */ + +H5FD_ros3_fapl_t anonymous_fa = { + H5FD__CURR_ROS3_FAPL_T_VERSION, + FALSE, "", "", "" }; +#endif /* H5_HAVE_ROS3_VFD */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_fapl_config_validation() + * + * Purpose: + * + * Test data consistency of fapl configuration. + * Tests `H5FD_ros3_validate_config` indirectly through `H5Pset_fapl_ros3`. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-10-23 + * + * Changes: None. + * + *--------------------------------------------------------------------------- + */ +static int +test_fapl_config_validation(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *msg; + herr_t expected; + H5FD_ros3_fapl_t config; + }; + + /************************ + * test-local variables * + ************************/ + +#ifdef H5_HAVE_ROS3_VFD + hid_t fapl_id = -1; /* file access property list ID */ + H5FD_ros3_fapl_t config; + H5FD_ros3_fapl_t fa_fetch; + herr_t success = SUCCEED; + unsigned int i = 0; + unsigned int ncases = 8; /* should equal number of cases */ + struct testcase *case_ptr = NULL; /* dumb work-around for possible */ + /* dynamic cases creation because */ + /* of compiler warnings Wlarger-than */ + struct testcase cases_arr[] = { + { "non-authenticating config allows empties.\n", + SUCCEED, + { H5FD__CURR_ROS3_FAPL_T_VERSION, /* version */ + FALSE, /* authenticate */ + "", /* aws_region */ + "", /* secret_id */ + "", /* secret_key */ + }, + }, + { "authenticating config asks for populated strings.\n", + FAIL, + { H5FD__CURR_ROS3_FAPL_T_VERSION, + TRUE, + "", + "", + "", + }, + }, + { "populated strings; key is the empty string?\n", + SUCCEED, + { H5FD__CURR_ROS3_FAPL_T_VERSION, + TRUE, + "region", + "me", + "", + }, + }, + { "id cannot be empty.\n", + FAIL, + { H5FD__CURR_ROS3_FAPL_T_VERSION, + TRUE, + "", + "me", + "", + }, + }, + { "region cannot be empty.\n", + FAIL, + { H5FD__CURR_ROS3_FAPL_T_VERSION, + TRUE, + "where", + "", + "", + }, + }, + { "all strings populated.\n", + SUCCEED, + { H5FD__CURR_ROS3_FAPL_T_VERSION, + TRUE, + "where", + "who", + "thisIsA GREAT seeeecrit", + }, + }, + { "incorrect version should fail\n", + FAIL, + { 12345, + FALSE, + "", + "", + "", + }, + }, + { "non-authenticating config cares not for (de)population" + "of strings.\n", + SUCCEED, + { H5FD__CURR_ROS3_FAPL_T_VERSION, + FALSE, + "someregion", + "someid", + "somekey", + }, + }, + }; + +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 fapl configuration validation"); + + /********* + * TESTS * + *********/ + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + for (i = 0; i < ncases; i++) { + + /*--------------- + * per-test setup + *--------------- + */ + case_ptr = &cases_arr[i]; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) /* sanity-check */ + + /*----------------------------------- + * Actually test. + * Mute stack trace in failure cases. + *----------------------------------- + */ + H5E_BEGIN_TRY { + /* `H5FD_ros3_validate_config(...)` is static/private + * to src/ros3.c and cannot (and should not?) be tested directly? + * Instead, validate config through public api. + */ + success = H5Pset_fapl_ros3(fapl_id, &case_ptr->config); + } H5E_END_TRY; + + JSVERIFY( case_ptr->expected, success, case_ptr->msg ) + + /* Make sure we can get back what we put in. + * Only valid if the fapl configuration does not result in error. + */ + if (success == SUCCEED) { + config = case_ptr->config; + JSVERIFY( SUCCEED, + H5Pget_fapl_ros3(fapl_id, &fa_fetch), + "unable to get fapl" ) + + JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, + fa_fetch.version, + "invalid version number" ) + JSVERIFY( config.version, + fa_fetch.version, + "version number mismatch" ) + JSVERIFY( config.authenticate, + fa_fetch.authenticate, + "authentication flag mismatch" ) + JSVERIFY_STR( config.aws_region, + fa_fetch.aws_region, + NULL ) + JSVERIFY_STR( config.secret_id, + fa_fetch.secret_id, + NULL ) + JSVERIFY_STR( config.secret_key, + fa_fetch.secret_key, + NULL ) + } + + /*----------------------------- + * per-test sanitation/teardown + *----------------------------- + */ + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + } /* for each test case */ + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fapl_id < 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + return 1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_fapl_config_validation */ + + +/*------------------------------------------------------------------------- + * + * Function: test_ros3_fapl() + * + * Purpose: Tests the file handle interface for the ROS3 driver + * + * As the ROS3 driver is 1) read only, 2) requires access + * to an S3 server (minio for now), this test is quite + * different from the other tests. + * + * For now, test only fapl & flags. Extend as the + * work on the VFD continues. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: John Mainzer + * 7/12/17 + * + * Changes: Test only fapl and flags. + * Jacob Smith 2017 + * + *------------------------------------------------------------------------- + */ +static int +test_ros3_fapl(void) +{ + /************************ + * test-local variables * + ************************/ + +#ifdef H5_HAVE_ROS3_VFD + hid_t fapl_id = -1; /* file access property list ID */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + H5FD_ros3_fapl_t ros3_fa_0 = { + H5FD__CURR_ROS3_FAPL_T_VERSION, /* version */ + FALSE, /* authenticate */ + "", /* aws_region */ + "", /* secret_id */ + "plugh", /* secret_key */ + }; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 fapl "); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + /* Set property list and file name for ROS3 driver. + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &ros3_fa_0) ) + + driver_id = H5Pget_driver(fapl_id); + FAIL_IF( driver_id < 0 ) + + /**************** + * Check that the VFD feature flags are correct + * SPEC MAY CHANGE + ******************/ + + FAIL_IF( H5FDdriver_query(driver_id, &driver_flags) < 0 ) + + JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE), + "bit(s) in `driver_flags` must align with " + "H5FD_FEAT_DATA_SIEVE" ) + + JSVERIFY( H5FD_FEAT_DATA_SIEVE, driver_flags, + "H5FD_FEAT_DATA_SIEVE should be the only supported flag") + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + + return 1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_ros3_fapl() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_vfd_open() + * + * Purpose: + * + * Demonstrate/specify VFD-level "Open" failure cases + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 1027-11-03 + * + *--------------------------------------------------------------------------- + */ +static int +test_vfd_open(void) +{ + /********************* + * test-local macros * + *********************/ + +#ifdef H5_HAVE_ROS3_VFD + +#define FAPL_H5P_DEFAULT -2 +#define FAPL_FILE_ACCESS -3 +#define FAPL_ROS3_ANON -4 + + /************************* + * test-local structures * + *************************/ + + struct test_condition { + const char *message; + const char *url; + unsigned flags; + int which_fapl; + haddr_t maxaddr; + }; + + /************************ + * test-local variables * + ************************/ + + struct test_condition tests[] = { + { "default property list (H5P_DEFAULT) is invalid", + url_text_public, + H5F_ACC_RDONLY, + FAPL_H5P_DEFAULT, + MAXADDR, + }, + { "generic file access property list is invalid", + url_text_public, + H5F_ACC_RDONLY, + FAPL_FILE_ACCESS, + MAXADDR, + }, + { "filename cannot be null", + NULL, + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "filename cannot be empty", + "", + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "filename must exist", + url_missing, + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "read-write flag not supported", + url_text_public, + H5F_ACC_RDWR, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "truncate flag not supported", + url_text_public, + H5F_ACC_TRUNC, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "create flag not supported", + url_text_public, + H5F_ACC_CREAT, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "EXCL flag not supported", + url_text_public, + H5F_ACC_EXCL, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "maxaddr cannot be 0 (caught in `H5FD_open()`)", + url_text_public, + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + 0, + }, + }; + H5FD_t *fd = NULL; + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; + hid_t fapl_file_access = -1; + unsigned i = 0; + unsigned tests_count = 10; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 VFD-level open"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + fapl_file_access = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_file_access < 0 ) + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &anonymous_fa) ) + + /********* + * TESTS * + *********/ + + /* all the test cases that will _not_ open + */ + for (i = 0; i < tests_count; i++) { + struct test_condition T = tests[i]; + hid_t _fapl_id = H5P_DEFAULT; + + fd = NULL; + + if (T.which_fapl == FAPL_FILE_ACCESS) + _fapl_id = fapl_file_access; + else if (T.which_fapl == FAPL_ROS3_ANON) + _fapl_id = fapl_id; + + H5E_BEGIN_TRY { + fd = H5FDopen(T.url, T.flags, _fapl_id, T.maxaddr); + } H5E_END_TRY; + if (NULL != fd) + JSVERIFY(1, 0, T.message); /* wrapper to print message and fail */ + } + + FAIL_IF( NULL != fd ) + + /* finally, show that a file can be opened + */ + fd = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + MAXADDR); + FAIL_IF( NULL == fd ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd) ) + fd = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_file_access) ) + fapl_file_access = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd) { + (void)H5FDclose(fd); + } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (fapl_file_access >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_file_access); + } H5E_END_TRY; + } + if (curl_ready == TRUE) { + curl_global_cleanup(); + } + + return 1; + +#undef FAPL_FILE_ACCESS +#undef FAPL_H5P_DEFAULT +#undef FAPL_ROS3_ANON + +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_vfd_open */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_eof_eoa() + * + * Purpose: + * + * Demonstrate behavior of get_eof, get_eoa, and set_eoa. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_eof_eoa(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + +#ifdef H5_HAVE_ROS3_VFD + H5FD_t *fd_shakespeare = NULL; + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 eof/eoa gets and sets"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + fd_shakespeare = H5FDopen( + url_text_restricted, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_shakespeare ) + + /********* + * TESTS * + *********/ + + /* verify as found + */ + JSVERIFY( 5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), NULL ) + JSVERIFY( H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW), + "mismatch between DEFAULT and RAW memory types" ) + JSVERIFY( 0, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA should be unset by H5FDopen" ) + + /* set EoA below EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202), + "unable to set EoA (lower)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 44442202, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /* set EoA above EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012), + "unable to set EoA (higher)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 6789012, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd_shakespeare) ) + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd_shakespeare) (void)H5FDclose(fd_shakespeare); + if (TRUE == curl_ready) curl_global_cleanup(); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_eof_eoa */ + + +/*----------------------------------------------------------------------------- + * + * Function: test_H5FDread_without_eoa_set_fails() + * + * Purpose: + * + * Demonstrate a not-obvious constraint by the library, preventing + * file read before EoA is set + * + * Programmer: Jacob Smith + * 2018-01-26 + * + *----------------------------------------------------------------------------- + */ +static int +test_H5FDread_without_eoa_set_fails(void) +{ +#ifdef H5_HAVE_ROS3_VFD + char buffer[256]; + unsigned int i = 0; + H5FD_t *file_shakespeare = NULL; + hid_t fapl_id = -1; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 VFD read-eoa temporal coupling library limitation "); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + /* create ROS3 fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + file_shakespeare = H5FDopen( + url_text_restricted, + H5F_ACC_RDONLY, + fapl_id, + MAXADDR); + FAIL_IF( NULL == file_shakespeare ) + + JSVERIFY( 0, H5FDget_eoa(file_shakespeare, H5FD_MEM_DEFAULT), + "EoA should remain unset by H5FDopen" ) + + for (i = 0; i < 256; i++) + buffer[i] = 0; /* zero buffer contents */ + + /******** + * TEST * + ********/ + + H5E_BEGIN_TRY { /* mute stack trace on expected failure */ + JSVERIFY( FAIL, + H5FDread(file_shakespeare, + H5FD_MEM_DRAW, + H5P_DEFAULT, + 1200699, + 102, + buffer), + "cannot read before eoa is set" ) + } H5E_END_TRY; + JSVERIFY_STR( "", buffer, "buffer should remain untouched" ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_shakespeare) ) + file_shakespeare = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (file_shakespeare) { (void)H5FDclose(file_shakespeare); } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_H5FDread_without_eoa_set_fails */ + + + +/*--------------------------------------------------------------------------- + * + * Function: test_read() + * + * Purpose: + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_read(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + struct testcase { + const char *message; /* purpose of test case */ + haddr_t eoa_set; /* set file EOA to this prior to read */ + size_t addr; /* offset of read in file */ + size_t len; /* length of read in file */ + herr_t success; /* expected return value of read function */ + const char *expected; /* expected contents of buffer; failure ignores */ + }; + + /************************ + * test-local variables * + ************************/ +#ifdef H5_HAVE_ROS3_VFD + struct testcase cases[] = { + { "successful range-get", + 6464, + 5691, + 32, /* fancy quotes are three bytes each(?) */ + SUCCEED, + "Quoth the Raven “Nevermore.”", + }, + { "read past EOA fails (EOA < EOF < addr)", + 3000, + 4000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA fails (EOA < addr < EOF < (addr+len))", + 3000, + 8000, + 100, + FAIL, + NULL, + }, + { "read past EOA/EOF fails ((EOA==EOF) < addr)", + 6464, + 7000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA/EOF fails (addr < (EOA==EOF) < (addr+len))", + 6464, + 6400, + 100, + FAIL, + NULL, + }, + { "read between EOF and EOA fails (EOF < addr < (addr+len) < EOA)", + 8000, + 7000, + 100, + FAIL, + NULL, + }, + }; + unsigned testcase_count = 6; + unsigned test_i = 0; + struct testcase test; + herr_t open_return = FAIL; + char buffer[S3_TEST_MAX_URL_SIZE]; + unsigned int i = 0; + H5FD_t *file_raven = NULL; + hid_t fapl_id = -1; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 VFD read/range-gets"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + /* create ROS3 fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + /* open file + */ + file_raven = H5FDopen( /* will open with "authenticating" fapl */ + url_text_public, /* TODO: check return state: anon access of restricted says OK? (not NULL) */ + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); /* Demonstrate success with "automatic" value */ + FAIL_IF( NULL == file_raven ) + + JSVERIFY( 6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), NULL ) + + /********* + * TESTS * + *********/ + + for (test_i = 0; test_i < testcase_count; test_i++) { + + /* -------------- * + * per-test setup * + * -------------- */ + + test = cases[test_i]; + open_return = FAIL; + + FAIL_IF( S3_TEST_MAX_URL_SIZE < test.len ) /* buffer too small! */ + + FAIL_IF( FAIL == + H5FD_set_eoa( file_raven, H5FD_MEM_DEFAULT, test.eoa_set) ) + + for (i = 0; i < S3_TEST_MAX_URL_SIZE; i++) /* zero buffer contents */ + buffer[i] = 0; + + /* ------------ * + * conduct test * + * ------------ */ + + H5E_BEGIN_TRY { + open_return = H5FDread( + file_raven, + H5FD_MEM_DRAW, + H5P_DEFAULT, + test.addr, + test.len, + buffer); + } H5E_END_TRY; + + JSVERIFY( test.success, + open_return, + test.message ) + if (open_return == SUCCEED) + JSVERIFY_STR( test.expected, buffer, NULL ) + + } /* for each testcase */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_raven) ) + file_raven = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (file_raven) + (void)H5FDclose(file_raven); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_read */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_noops_and_autofails() + * + * Purpose: + * + * Demonstrate the unavailable and do-nothing routines unique to + * Read-Only VFD. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_noops_and_autofails(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + +#ifdef H5_HAVE_ROS3_VFD + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; + H5FD_t *file = NULL; + const char data[36] = "The Force shall be with you, always"; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 VFD always-fail and no-op routines"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + /* create ROS3 fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + JSVERIFY( SUCCEED, H5Pset_fapl_ros3(fapl_id, &anonymous_fa), NULL ) + + /* open file + */ + file = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == file ) + + /********* + * TESTS * + *********/ + + /* auto-fail calls to write and truncate + */ + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data), + "write must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, FALSE), + "truncate must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, TRUE), + "truncate must fail (closing)" ) + } H5E_END_TRY; + + /* no-op calls to `lock()` and `unlock()` + */ + JSVERIFY( SUCCEED, + H5FDlock(file, TRUE), + "lock always succeeds; has no effect" ) + JSVERIFY( SUCCEED, + H5FDlock(file, FALSE), + NULL ) + JSVERIFY( SUCCEED, + H5FDunlock(file), + NULL ) + /* Lock/unlock with null file or similar error crashes tests. + * HDassert in calling heirarchy, `H5FD[un]lock()` and `H5FD_[un]lock()` + */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file) ) + file = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file) { (void)H5FDclose(file); } + if (curl_ready == TRUE) { curl_global_cleanup(); } + + return 1; + +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_noops_and_autofails*/ + + +/*--------------------------------------------------------------------------- + * + * Function: test_cmp() + * + * Purpose: + * + * Verify "file comparison" behavior. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_cmp(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + +#ifdef H5_HAVE_ROS3_VFD + H5FD_t *fd_raven = NULL; + H5FD_t *fd_shakes = NULL; + H5FD_t *fd_raven_2 = NULL; + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("ROS3 cmp (comparison)"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + JSVERIFY( SUCCEED, H5Pset_fapl_ros3(fapl_id, &restricted_access_fa), NULL ) + + fd_raven = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_raven ) + + fd_shakes = H5FDopen( + url_text_restricted, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_shakes ) + + fd_raven_2 = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_raven_2 ) + + /********* + * TESTS * + *********/ + + JSVERIFY( 0, H5FDcmp(fd_raven, fd_raven_2), NULL ) + JSVERIFY( -1, H5FDcmp(fd_raven, fd_shakes), NULL ) + JSVERIFY( -1, H5FDcmp(fd_shakes, fd_raven_2), NULL ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd_raven) ) + fd_raven = NULL; + FAIL_IF( FAIL == H5FDclose(fd_shakes) ) + fd_shakes = NULL; + FAIL_IF( FAIL == H5FDclose(fd_raven_2) ) + fd_raven_2 = NULL; + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd_raven != NULL) (void)H5FDclose(fd_raven); + if (fd_raven_2 != NULL) (void)H5FDclose(fd_raven_2); + if (fd_shakes != NULL) (void)H5FDclose(fd_shakes); + if (TRUE == curl_ready) curl_global_cleanup(); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_cmp */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_H5F_integration() + * + * Purpose: + * + * Demonstrate S3 file-open through H5F API. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-07 + * + *--------------------------------------------------------------------------- + */ +static int +test_H5F_integration(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + +#ifdef H5_HAVE_ROS3_VFD + hid_t file = -1; + hid_t fapl_id = -1; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("S3 file access through HD5F library (H5F API)"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; +#else + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + /********* + * TESTS * + *********/ + + /* Read-Write Open access is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fopen( + url_h5_public, + H5F_ACC_RDWR, + fapl_id) ) + } H5E_END_TRY; + + /* H5Fcreate() is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fcreate( + url_missing, + H5F_ACC_RDONLY, + H5P_DEFAULT, + fapl_id) ) + } H5E_END_TRY; + + /* Successful open. + */ + file = H5Fopen( + url_h5_public, + H5F_ACC_RDONLY, + fapl_id); + FAIL_IF( file < 0 ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5Fclose(file) ) + file = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ +HDprintf("\nerror!"); fflush(stdout); + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file > 0) + (void)H5Fclose(file); + + return 1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_H5F_integration */ + + + + +/*------------------------------------------------------------------------- + * + * Function: main + * + * Purpose: Tests the basic features of Virtual File Drivers + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + +#ifdef H5_HAVE_ROS3_VFD + const char *bucket_url_env = NULL; + + /************************ + * initialize test urls * + ************************/ + + bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL"); + if (bucket_url_env == NULL || bucket_url_env[0] == '\0') { + HDprintf("WARNING: S3 bucket url is not defined in enviornment " \ + "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n"); + } else { + HDstrncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE); + s3_test_bucket_defined = TRUE; + } + + if (S3_TEST_MAX_URL_SIZE < snprintf( + url_text_restricted, + (size_t)S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_TEXT_RESTRICTED)) + { + HDprintf("* ros3 setup failed (text_restricted) ! *\n"); + return 1; + } + if (S3_TEST_MAX_URL_SIZE < HDsnprintf( + url_text_public, + (size_t)S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_TEXT_PUBLIC)) + { + HDprintf("* ros3 setup failed (text_public) ! *\n"); + return 1; + } + if (S3_TEST_MAX_URL_SIZE < HDsnprintf( + url_h5_public, + (size_t)S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_H5_PUBLIC)) + { + HDprintf("* ros3 setup failed (h5_public) ! *\n"); + return 1; + } + if (S3_TEST_MAX_URL_SIZE < HDsnprintf( + url_missing, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_MISSING)) + { + HDprintf("* ros3 setup failed (missing) ! *\n"); + return 1; + } + + /************************************** + * load credentials and prepare fapls * + **************************************/ + + /* "clear" profile data strings */ + s3_test_aws_access_key_id[0] = '\0'; + s3_test_aws_secret_access_key[0] = '\0'; + s3_test_aws_region[0] = '\0'; + + /* attempt to load test credentials + * if unable, certain tests will be skipped + */ + if (SUCCEED == H5FD_s3comms_load_aws_profile( + S3_TEST_PROFILE_NAME, + s3_test_aws_access_key_id, + s3_test_aws_secret_access_key, + s3_test_aws_region)) + { + s3_test_credentials_loaded = 1; + HDstrncpy(restricted_access_fa.aws_region, + (const char *)s3_test_aws_region, + H5FD__ROS3_MAX_REGION_LEN); + HDstrncpy(restricted_access_fa.secret_id, + (const char *)s3_test_aws_access_key_id, + H5FD__ROS3_MAX_SECRET_ID_LEN); + HDstrncpy(restricted_access_fa.secret_key, + (const char *)s3_test_aws_secret_access_key, + H5FD__ROS3_MAX_SECRET_KEY_LEN); + } +#endif /* H5_HAVE_ROS3_VFD */ + + /****************** + * commence tests * + ******************/ + + h5_reset(); + + HDprintf("Testing ros3 VFD functionality.\n"); + + nerrors += test_fapl_config_validation(); + nerrors += test_ros3_fapl(); + nerrors += test_vfd_open(); + nerrors += test_eof_eoa(); + nerrors += test_H5FDread_without_eoa_set_fails(); + nerrors += test_read(); + nerrors += test_noops_and_autofails(); + nerrors += test_cmp(); + nerrors += test_H5F_integration(); + + if (nerrors > 0) { + HDprintf("***** %d ros3 TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + nerrors = 1; + } else { + HDprintf("All ros3 tests passed.\n"); + } + return nerrors; /* 0 if no errors, 1 if any errors */ + +} /* main() */ + + diff --git a/test/s3comms.c b/test/s3comms.c new file mode 100644 index 0000000..a797e04 --- /dev/null +++ b/test/s3comms.c @@ -0,0 +1,2813 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Purpose: Unit tests for the S3 Communications (s3comms) module. + * + * Programmer: Jacob Smith <jake.smith@hdfgroup.org> + * 2017-10-11 + */ + +#include "h5test.h" +#include "H5FDs3comms.h" +#include "H5MMprivate.h" /* memory management */ + +/***************************************************************************** + * + * FILE-LOCAL TESTING MACROS + * + * Purpose: + * + * 1) Upon test failure, goto-jump to single-location teardown in test + * function. E.g., `error:` (consistency with HDF corpus) or + * `failed:` (reflects purpose). + * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. + * 2) Increase clarity and reduce overhead found with `TEST_ERROR`. + * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:" + * requires reading of entire line to know whether this if/call is + * part of the test setup, test operation, or a test unto itself. + * 3) Provide testing macros with optional user-supplied failure message; + * if not supplied (NULL), generate comparison output in the spirit of + * test-driven development. E.g., "expected 5 but was -3" + * User messages clarify test's purpose in code, encouraging description + * without relying on comments. + * 4) Configurable expected-actual order in generated comparison strings. + * Some prefer `VERIFY(expected, actual)`, others + * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch + * to satifsy both parties, assuming one paradigm per test file. + * (One could #undef and redefine the flag through the file as desired, + * but _why_.) + * Provided as courtesy, per consideration for inclusion in the library + * proper. + * + * Macros: + * + * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order + * FAIL_IF() - check condition + * FAIL_UNLESS() - check _not_ condition + * JSVERIFY() - long-int equality check; prints reason/comparison + * JSVERIFY_NOT() - long-int inequality check; prints + * JSVERIFY_STR() - string equality check; prints + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * ifdef flag: JSVERIFY_EXP_ACT + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_EXP_ACT 1L + + +/*---------------------------------------------------------------------------- + * + * Macro: JSFAILED_AT() + * + * Purpose: + * + * Preface a test failure by printing "*FAILED*" and location to stdout + * Similar to `H5_FAILED(); AT();` from h5test.h + * + * *FAILED* at somefile.c:12 in function_name()... + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSFAILED_AT() { \ + HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_IF() + * + * Purpose: + * + * Make tests more accessible and less cluttered than + * `if (thing == otherthing()) TEST_ERROR` + * paradigm. + * + * The following lines are roughly equivalent: + * + * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests) + * `FAIL_IF(myfunc() < 0)` + * + * Prints a generic "FAILED AT" line to stdout and jumps to `error`, + * similar to `TEST_ERROR` in h5test.h + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_IF(condition) \ +if (condition) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_UNLESS() + * + * Purpose: + * + * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", + * e.g., "a != b". + * + * Opposite of FAIL_IF; fails if the given condition is _not_ true. + * + * `FAIL_IF( 5 != my_op() )` + * is equivalent to + * `FAIL_UNLESS( 5 == my_op() )` + * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. + * (see JSVERIFY) + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_UNLESS(condition) \ +if (!(condition)) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_LONG() + * + * Purpose: + * + * Print an failure message for long-int arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:488 in somefunc()... + * forest must be made of trees. + * + * or + * + * *FAILED* at myfile.c:488 in somefunc()... + * ! Expected 425 + * ! Actual 3 + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_LONG(expected, actual, reason) { \ + JSFAILED_AT() \ + if (reason!= NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf(" ! Expected %ld\n ! Actual %ld\n", \ + (long)(expected), (long)(actual)); \ + } \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_STR() + * + * Purpose: + * + * Print an failure message for string arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:421 in myfunc()... + * Blue and Red strings don't match! + * + * or + * + * *FAILED* at myfile.c:421 in myfunc()... + * !!! Expected: + * this is my expected + * string + * !!! Actual: + * not what I expected at all + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_STR(expected, actual, reason) { \ + JSFAILED_AT() \ + if ((reason) != NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \ + (expected), (actual)); \ + } \ +} + +#ifdef JSVERIFY_EXP_ACT +/* VERIFY rountines with paramter order (<expected>, <actual> [, <msg> ]) + */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY() + * + * Purpose: + * + * Verify that two long integers are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(expected, actual, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY */ + +#if 0 /* UNUSED */ + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_NOT() + * + * Purpose: + * + * Verify that two long integers are _not_ equal. + * If equal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(expected, actual, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ +#endif /* JSVERIFY_NOT unused */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_STR() + * + * Purpose: + * + * Verify that two strings are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(expected, actual, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + + +#else +/* JSVERIFY_EXP_ACT not defined + * + * Repeats macros above, but with actual/expected parameters reversed. + */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY() + * See: JSVERIFY documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(actual, expected, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY */ + +#if 0 /* UNUSED */ + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_NOT() + * See: JSVERIFY_NOT documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(actual, expected, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ +#endif /* JSVERIFY_NOT unused */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_STR() + * See: JSVERIFY_STR documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(actual, expected, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + +#endif /* ifdef/else JSVERIFY_EXP_ACT */ + + +#ifdef H5_HAVE_ROS3_VFD + +#define S3_TEST_PROFILE_NAME "ros3_vfd_test" + +#define S3_TEST_RESOURCE_TEXT_RESTRICTED "t8.shakespeare.txt" +#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt" +#define S3_TEST_RESOURCE_MISSING "missing.csv" + +#define S3_TEST_RUN_TIMEOUT 0 /* run tests that might hang */ +#define S3_TEST_MAX_URL_SIZE 256 /* char array size */ + +/* Global variables for aws test profile. + * An attempt is made to read ~/.aws/credentials and ~/.aws/config upon test + * startup -- if unable to open either file or cannot load region, id, and key, + * tests connecting with S3 will not be run + */ +static int s3_test_credentials_loaded = 0; +static char s3_test_aws_region[16] = ""; +static char s3_test_aws_access_key_id[64] = ""; +static char s3_test_aws_secret_access_key[128] = ""; +static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; +static hbool_t s3_test_bucket_defined = FALSE; + +#endif /* H5_HAVE_ROS3_VFD */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_macro_format_credential() + * + * Purpose: + * + * Demonstrate that the macro `S3COMMS_FORMAT_CREDENTIAL` + * performs as expected. + * + * Programmer: Jacob Smith + * 2017-09-19 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_macro_format_credential(void) +{ + /************************ + * test-local variables * + ************************/ + +#ifdef H5_HAVE_ROS3_VFD + char dest[256]; + const char access[] = "AKIAIOSFODNN7EXAMPLE"; + const char date[] = "20130524"; + const char region[] = "us-east-1"; + const char service[] = "s3"; + const char expected[] = + "AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("test_macro_format_credential"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + FAIL_IF( S3COMMS_MAX_CREDENTIAL_SIZE < + S3COMMS_FORMAT_CREDENTIAL(dest, access, date, region, service) ) + + JSVERIFY_STR( expected, dest, NULL ) + + PASSED(); + return 0; +#endif /* H5_HAVE_ROS3_VFD */ + +error: + return -1; + +} /* test_macro_format_credential */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_aws_canonical_request() + * + * Purpose: + * + * Demonstrate the construction of a Canoncial Request (and Signed Headers) + * + * Elided / not yet implemented: + * Query strings + * request "body" + * + * Programmer: Jacob Smith + * 2017-10-04 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_aws_canonical_request(void) +{ + /************************* + * test-local structures * + *************************/ + +#ifdef H5_HAVE_ROS3_VFD + struct header { + const char *name; + const char *value; + }; + + struct testcase { + const char *exp_request; + const char *exp_headers; + const char *verb; + const char *resource; + unsigned int listsize; + struct header list[5]; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "GET\n/some/path.file\n\nhost:somebucket.someserver.somedomain\nrange:bytes=150-244\n\nhost;range\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "host;range", + "GET", + "/some/path.file", + 2, + { {"Range", "bytes=150-244"}, + {"Host", "somebucket.someserver.somedomain"}, + }, + }, + { "HEAD\n/bucketpath/myfile.dat\n\nhost:place.domain\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:19411207T150803Z\n\nhost;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "host;x-amz-content-sha256;x-amz-date", + "HEAD", + "/bucketpath/myfile.dat", + 3, + { {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {"host", "place.domain"}, + {"x-amz-date", "19411207T150803Z"}, + } + }, + { "PUT\n/\n\n\n\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "", + "PUT", + "/", + 0, + {{"",""},}, /* unused; satisfies compiler */ + }, + }; /* struct testcase cases[] */ + struct testcase *C = NULL; + char cr_dest[512]; /* canonical request */ + hrb_t *hrb = NULL; /* http request buffer object */ + unsigned int i = 0; /* looping/indexing */ + unsigned int j = 0; /* looping/indexing */ + hrb_node_t *node = NULL; /* http headers list pointer */ + unsigned int n_cases = 3; + char sh_dest[64]; /* signed headers */ +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("test_aws_canonical_request"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + for (i = 0; i < n_cases; i++) { + /* pre-test bookkeeping + */ + C = &cases[i]; + for (j = 0; j < 256; j++) { cr_dest[j] = 0; } /* zero request buffer */ + for (j = 0; j < 64; j++) { sh_dest[j] = 0; } /* zero headers buffer */ + + /* create HTTP request object with given verb, resource/path + */ + hrb = H5FD_s3comms_hrb_init_request(C->verb, + C->resource, + "HTTP/1.1"); + HDassert(hrb->body == NULL); + + /* Create headers list from test case input + */ + for (j = 0; j < C->listsize; j++) { + FAIL_IF( FAIL == + H5FD_s3comms_hrb_node_set( + &node, + C->list[j].name, + C->list[j].value)); + } + + hrb->first_header = node; + + /* test + */ + JSVERIFY( SUCCEED, + H5FD_s3comms_aws_canonical_request(cr_dest, sh_dest, hrb), + " unable to compose canonical request" ) + JSVERIFY_STR( C->exp_headers, sh_dest, NULL ) + JSVERIFY_STR( C->exp_request, cr_dest, NULL ) + + /* tear-down + */ + while (node != NULL) + FAIL_IF( FAIL == + H5FD_s3comms_hrb_node_set(&node, node->name, NULL)); + HDassert(NULL == node); + FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb)); + HDassert(NULL == hrb); + + } /* for each test case */ + + /*************** + * ERROR CASES * + ***************/ + + /* malformed hrb and/or node-list + */ + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(cr_dest, sh_dest, NULL), + "http request object cannot be null" ) + + hrb = H5FD_s3comms_hrb_init_request("GET", "/", "HTTP/1.1"); + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(NULL, sh_dest, hrb), + "canonical request destination cannot be NULL" ) + + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(cr_dest, NULL, hrb), + "signed headers destination cannot be null" ) + + FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb) ) + HDassert( NULL == hrb ); + + PASSED(); + return 0; + +error: + + if (node != NULL) { + while (node != NULL) + (void)H5FD_s3comms_hrb_node_set(&node, node->name, NULL); + HDassert( node == NULL ); + } + if (hrb != NULL) + (void)H5FD_s3comms_hrb_destroy(&hrb); + + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_aws_canonical_request */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_bytes_to_hex + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_bytes_to_hex()`. + * + * Return: + * + * Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_bytes_to_hex(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char exp[17]; /* in size * 2 + 1 for null terminator */ + const unsigned char in[8]; + size_t size; + hbool_t lower; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "52F3000C9A", + {82,243,0,12,154}, + 5, + FALSE, + }, + { "009a0cf3005200", /* lowercase alphas */ + {0,154,12,243,0,82,0}, + 7, + TRUE, + }, + { "", + {17,63,26,56}, + 0, + FALSE, /* irrelevant */ + }, + }; + int i = 0; + int n_cases = 3; + char out[17]; + int out_off = 0; + + + + TESTING("bytes-to-hex"); + + for (i = 0; i < n_cases; i++) { + for (out_off = 0; out_off < 17; out_off++) { + out[out_off] = 0; + } + + JSVERIFY( SUCCEED, + H5FD_s3comms_bytes_to_hex(out, + cases[i].in, + cases[i].size, + cases[i].lower), + NULL ) + + JSVERIFY_STR(cases[i].exp, out, NULL) + } + + /* dest cannot be null + */ + JSVERIFY( FAIL, + H5FD_s3comms_bytes_to_hex( + NULL, + (const unsigned char *)"nada", + 5, + FALSE), + "destination cannot be null" ) + + PASSED(); + return 0; + +error: + return -1; + +} /* test_bytes_to_hex */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_hrb_init_request() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_hrb_init_request()` + * + * Programmer: Jacob Smith + * 2017-09-20 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_hrb_init_request(void) +{ +#ifdef H5_HAVE_ROS3_VFD + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char msg[64]; + const char *verb; + const char *resource; + const char *exp_res; + const char *version; + hbool_t ret_null; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "get HTTP request just as we provided", + "GET", + "/path/to/some/file", + "/path/to/some/file", + "HTTP/1.1", + FALSE, + }, + { "null verb substitues to GET", + NULL, + "/MYPATH/MYFILE.tiff", + "/MYPATH/MYFILE.tiff", + "HTTP/1.1", + FALSE, + }, + { "demonstrate non-GET verb", + "HEAD", + "/MYPATH/MYFILE.tiff", + "/MYPATH/MYFILE.tiff", + "HTTP/1.1", + FALSE, + }, + { "slash prepented to resource path, if necessary", + NULL, + "MYPATH/MYFILE.tiff", + "/MYPATH/MYFILE.tiff", + NULL, + FALSE, + }, + { "null resource path causes problem", + "GET", + NULL, + NULL, + NULL, + TRUE, + }, + }; + struct testcase *C = NULL; + unsigned int i = 0; + unsigned int ncases = 5; + hrb_t *req = NULL; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("hrb_init_request"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + for (i = 0; i < ncases; i++) { + C = &cases[i]; + req = H5FD_s3comms_hrb_init_request(C->verb, + C->resource, + C->version); + if (cases[i].ret_null == TRUE) { + FAIL_IF( req != NULL ); + } else { + FAIL_IF( req == NULL ); + JSVERIFY( S3COMMS_HRB_MAGIC, req->magic, NULL ) + if (C->verb == NULL) { + JSVERIFY_STR( "GET", req->verb, NULL ) + } else { + JSVERIFY_STR( req->verb, C->verb, NULL ) + } + JSVERIFY_STR( "HTTP/1.1", req->version, NULL ) + JSVERIFY_STR( C->exp_res, req->resource, NULL ) + FAIL_IF( req->first_header != NULL ); + FAIL_IF( req->body != NULL ); + JSVERIFY( 0, req->body_len, NULL ) + JSVERIFY( SUCCEED, H5FD_s3comms_hrb_destroy(&req), + "unable to destroy hrb_t" ) + FAIL_IF( NULL != req ); /* should annull pointer as well as free */ + } + + } /* for each testcase */ + + PASSED(); + return 0; + +error: + (void)H5FD_s3comms_hrb_destroy(&req); + + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_hrb_init_request */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_hrb_node_set() + * + * Purpose: + * + * Test operations on hrb_node_t structure + * + * Programmer: Jacob Smith + * 2017-09-22 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_hrb_node_set(void) +{ +#ifdef H5_HAVE_ROS3_VFD + /************************* + * test-local structures * + *************************/ + + /* bundle of name/value representing an hrb_node_t + */ + typedef struct node_mock_t { + const char *name; + const char *value; + } node_mock_t; + + /* bundle for a testcase + * + * `message` + * purpose of the testcase + * + * `delta` + * container for name and value strings to pass into node-set function + * to to modify the list. + * + * `returned` + * expected return value of node-set function + * + * `given` + * `expected` + * string arrays representing the state of the list before and after + * modification. The number of strings must be even, with each name + * paired to a value. `NULL` terminates the list, with `{NULL}` + * representing the empty list. + */ + typedef struct testcase { + const char *message; + node_mock_t delta; + herr_t returned; + const char *given[11]; /* name/value pairs in array; NULL sentinel */ + const char *expected[11]; + } testcase; + + /************************ + * test-local variables * + ************************/ + + testcase cases[] = { + { "cannot remove node from null list", + { "Host", NULL }, + FAIL, + {NULL}, + {NULL}, + }, + { "cannot create list with NULL field name", + { NULL, "somevalue" }, + FAIL, + {NULL}, + {NULL}, + }, + { "create a new list", + { "Host", "somevalue" }, + SUCCEED, + {NULL}, + { "Host", "somevalue", + NULL, + }, + }, + { "insert new node at head list", + { "Host", "somevalue" }, + SUCCEED, + { "Range", "bytes=20-40", + NULL, + }, + { "Host", "somevalue", + "Range", "bytes=20-40", + NULL, + }, + }, + { "append new node at list end", + { "x-amz-date", "somevalue" }, + SUCCEED, + { "Range", "bytes=20-40", + NULL, + }, + { "Range", "bytes=20-40", + "x-amz-date", "somevalue", + NULL, + }, + }, + { "insert new node inside list", + { "Intermediary", "somevalue" }, + SUCCEED, + { "Host", "somehost" , + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "Intermediary", "somevalue", + "Range", "bytes=20-40", + NULL, + }, + }, + { "modify node", + { "Range", "bytes=40-80" }, + SUCCEED, + { "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "Range", "bytes=40-80", + NULL, + }, + }, + { "modify node with new case", + { "RANGE", "bytes=40-80" }, + SUCCEED, + { "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "RANGE", "bytes=40-80", + NULL, + }, + }, + { "cannot add node with no name", + { NULL, "bytes=40-80" }, + FAIL, + { "Host", "somehost", + NULL, + }, + { "Host", "somehost", + NULL, + }, + }, + { "add node with 'empty' name", + { "", "bytes=40-80" }, + SUCCEED, + { "Host", "somehost", + NULL, + }, + { "", "bytes=40-80", + "Host", "somehost", + NULL, + }, + }, + { "remove node from end of list", + { "Host", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + NULL, + }, + }, + { "remove node from middle of list", + { "Host", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + }, + { "remove node from start of list", + { "Date", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + }, + { "remove only node in list", + { "Date", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + NULL, + }, + { NULL, + }, + }, + { "attempt to remove absent node fails", + { "Host", NULL }, + FAIL, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + }, + { "removal is case-insensitive", + { "hOsT", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + }, + }; + unsigned testcases_count = 16; + unsigned test_i = 0; + + hrb_node_t *list = NULL; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("test_hrb_node_t"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + for (test_i = 0; test_i < testcases_count; test_i++) { + const hrb_node_t *node = NULL; + const testcase *test = &(cases[test_i]); + unsigned mock_i = 0; + + /********* + * SETUP * + *********/ + + for (mock_i = 0; test->given[mock_i] != NULL; mock_i += 2) { + const char *name = test->given[mock_i]; + const char *valu = test->given[mock_i+1]; + + FAIL_IF( SUCCEED != + H5FD_s3comms_hrb_node_set(&list, name, valu) ) + } + /******** + * TEST * + ********/ + + /* perform modification on list + */ + JSVERIFY( test->returned, + H5FD_s3comms_hrb_node_set(&list, + test->delta.name, + test->delta.value), + test->message ) + + + /* verify resulting list + */ + node = list; + mock_i = 0; + while (test->expected[mock_i] != NULL && node != NULL) { + const char *name = test->expected[mock_i]; + const char *valu = test->expected[mock_i+1]; + + JSVERIFY_STR( name, node->name, NULL ) + JSVERIFY_STR( valu, node->value, NULL ) + + mock_i += 2; + node = node->next; + } + FAIL_IF( test->expected[mock_i] != NULL ) + FAIL_IF( node != NULL ) + + /************ + * TEARDOWN * + ************/ + + while (list != NULL) { + FAIL_IF( SUCCEED != + H5FD_s3comms_hrb_node_set(&list, list->name, NULL) ) + } + } + + PASSED(); + return 0; + +error: + while (list != NULL) + (void)H5FD_s3comms_hrb_node_set(&list, list->name, NULL); + + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_hrb_node_t */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_HMAC_SHA256() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_HMAC_SHA256()` + * + * Programmer: Jacob Smith + * 2017-09-19 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_HMAC_SHA256(void) +{ + + /************************* + * test-local structures * + *************************/ + +#ifdef H5_HAVE_ROS3_VFD + struct testcase { + herr_t ret; /* SUCCEED/FAIL expected from call */ + const unsigned char key[SHA256_DIGEST_LENGTH]; + size_t key_len; + const char *msg; + size_t msg_len; + const char *exp; /* not used if ret == FAIL */ + size_t dest_size; /* if 0, `dest` is not malloc'd */ + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { SUCCEED, + { 0xdb, 0xb8, 0x93, 0xac, 0xc0, 0x10, 0x96, 0x49, + 0x18, 0xf1, 0xfd, 0x43, 0x3a, 0xdd, 0x87, 0xc7, + 0x0e, 0x8b, 0x0d, 0xb6, 0xbe, 0x30, 0xc1, 0xfb, + 0xea, 0xfe, 0xfa, 0x5e, 0xc6, 0xba, 0x83, 0x78, + }, + SHA256_DIGEST_LENGTH, + "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972", + HDstrlen("AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972"), + "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", + SHA256_DIGEST_LENGTH * 2 + 1, /* +1 for null terminator */ + }, + { SUCCEED, + {'J','e','f','e'}, + 4, + "what do ya want for nothing?", + 28, + "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", + SHA256_DIGEST_LENGTH * 2 + 1, + }, + { FAIL, + "DOESN'T MATTER", + 14, + "ALSO IRRELEVANT", + 15, + NULL, + 0, /* dest -> null, resulting in immediate error */ + }, + }; + char *dest = NULL; + int i = 0; + int n_cases = 3; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("HMAC_SHA256"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + for (i = 0; i < n_cases; i++) { + if (cases[i].dest_size == 0) { + dest = NULL; + } else { + dest = (char *)HDmalloc(sizeof(char) * cases[i].dest_size); + HDassert(dest != NULL); + } + + JSVERIFY( cases[i].ret, + H5FD_s3comms_HMAC_SHA256( + cases[i].key, + cases[i].key_len, + cases[i].msg, + cases[i].msg_len, + dest), + cases[i].msg ); + if (cases[i].ret == SUCCEED) { +#ifdef VERBOSE + if (0 != + strncmp(cases[i].exp, + dest, + HDstrlen(cases[i].exp))) + { + /* print out how wrong things are, and then fail + */ + dest = (char *)realloc(dest, cases[i].dest_size + 1); + HDassert(dest != NULL); + dest[cases[i].dest_size] = 0; + HDfprintf(stdout, + "ERROR:\n!!! \"%s\"\n != \"%s\"\n", + cases[i].exp, + dest); + TEST_ERROR; + } +#else + /* simple pass/fail test + */ + JSVERIFY( 0, + strncmp(cases[i].exp, dest, HDstrlen(cases[i].exp)), + NULL); +#endif + } + free(dest); + } + + PASSED(); + return 0; + +error: + free(dest); + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_HMAC_SHA256 */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_nlowercase() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_nlowercase()` + * + * Programmer: Jacob Smith + * 2017-19-18 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_nlowercase(void) +{ + /************************* + * test-local structures * + *************************/ + +#ifdef H5_HAVE_ROS3_VFD + struct testcase { + const char *in; + size_t len; + const char *exp; + }; + + /************************ + * test-local variables * + ************************/ + + /* any character after in exp on or after exp[len] is undefined. + * in this test, kept as the null character for simplicity. + */ + struct testcase cases[] = { + { "HALlEluJAh", + 6, + "hallel", + }, + { "all\0 lower", + 10, + "all\0 lower", + }, + { "to meeeeeee", + 0, + "", + }, + }; + char *dest = NULL; + int i = 0; + int n_cases = 3; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("nlowercase"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + for (i = 0; i < n_cases; i++) { + dest = (char *)HDmalloc(sizeof(char) * 16); + + JSVERIFY( SUCCEED, + H5FD_s3comms_nlowercase(dest, + cases[i].in, + cases[i].len), + cases[i].in ) + if (cases[i].len > 0) { + JSVERIFY( 0, strncmp(dest, cases[i].exp, cases[i].len), NULL ) + } + free(dest); + } + + JSVERIFY( FAIL, + H5FD_s3comms_nlowercase(NULL, + cases[0].in, + cases[0].len), + "null distination should fail" ) + + PASSED(); + return 0; + +error: + free(dest); + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_nlowercase */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_parse_url() + * + * Programmer: Jacob Smith + * 2017-11-?? + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_parse_url(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + +#ifdef H5_HAVE_ROS3_VFD + typedef struct { + const char *scheme; + const char *host; + const char *port; + const char *path; + const char *query; + } const_purl_t; + + struct testcase { + const char *url; + herr_t exp_ret; /* expected return; */ + /* if FAIL, `expected` is unused */ + const_purl_t expected; + const char *msg; + }; + + /************************ + * test-local variables * + ************************/ + + parsed_url_t *purl = NULL; + unsigned int i = 0; + unsigned int ncases = 15; + struct testcase cases[] = { + { NULL, + FAIL, + { NULL, NULL, NULL, NULL, NULL }, + "null url", + }, + { "", + FAIL, + { NULL, NULL, NULL, NULL, NULL }, + "empty url", + }, + { "ftp://[1000:4000:0002:2010]", + SUCCEED, + { "ftp", + "[1000:4000:0002:2010]", + NULL, + NULL, + NULL, + }, + "IPv6 ftp and empty path (root)", + }, + { "ftp://[1000:4000:0002:2010]:2040", + SUCCEED, + { "ftp", + "[1000:4000:0002:2010]", + "2040", + NULL, + NULL, + }, + "root IPv6 ftp with port", + }, + { "http://some.domain.org:9000/path/to/resource.txt", + SUCCEED, + { "http", + "some.domain.org", + "9000", + "path/to/resource.txt", + NULL, + }, + "without query", + }, + { "https://domain.me:00/file.txt?some_params unchecked", + SUCCEED, + { "https", + "domain.me", + "00", + "file.txt", + "some_params unchecked", + }, + "with query", + }, + { "ftp://domain.com/", + SUCCEED, + { "ftp", + "domain.com", + NULL, + NULL, + NULL, + }, + "explicit root w/out port", + }, + { "ftp://domain.com:1234/", + SUCCEED, + { "ftp", + "domain.com", + "1234", + NULL, + NULL, + }, + "explicit root with port", + }, + { "ftp://domain.com:1234/file?", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "empty query is invalid", + }, + { "ftp://:1234/file", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "no host", + }, + { "h&r block", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "no scheme (bad URL)", + }, + { "http://domain.com?a=b&d=b", + SUCCEED, + { "http", + "domain.com", + NULL, + NULL, + "a=b&d=b", + }, + "QUERY with implict PATH", + }, + { "http://[5]/path?a=b&d=b", + SUCCEED, + { "http", + "[5]", + NULL, + "path", + "a=b&d=b", + }, + "IPv6 extraction is really dumb", + }, + { "http://[1234:5678:0910:1112]:port/path", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "non-decimal PORT (port)", + }, + { "http://mydomain.com:01a3/path", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "non-decimal PORT (01a3)", + }, + }; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("url-parsing functionality"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + /********* + * TESTS * + *********/ + + for (i = 0; i < ncases; i++) { + HDassert( purl == NULL ); + + JSVERIFY( cases[i].exp_ret, + H5FD_s3comms_parse_url(cases[i].url, &purl), + cases[i].msg ) + + if (cases[i].exp_ret == FAIL) { + /* on FAIL, `purl` should be untouched--remains NULL */ + FAIL_UNLESS( purl == NULL ) + } else { + /* on SUCCEED, `purl` should be set */ + FAIL_IF( purl == NULL ) + + if (cases[i].expected.scheme != NULL) { + FAIL_IF( NULL == purl->scheme ) + JSVERIFY_STR( cases[i].expected.scheme, + purl->scheme, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->scheme ) + } + + if (cases[i].expected.host != NULL) { + FAIL_IF( NULL == purl->host ) + JSVERIFY_STR( cases[i].expected.host, + purl->host, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->host ) + } + + if (cases[i].expected.port != NULL) { + FAIL_IF( NULL == purl->port ) + JSVERIFY_STR( cases[i].expected.port, + purl->port, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->port ) + } + + if (cases[i].expected.path != NULL) { + FAIL_IF( NULL == purl->path ) + JSVERIFY_STR( cases[i].expected.path, + purl->path, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->path ) + } + + if (cases[i].expected.query != NULL) { + FAIL_IF( NULL == purl->query ) + JSVERIFY_STR( cases[i].expected.query, + purl->query, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->query ) + } + } /* if parse-url return SUCCEED/FAIL */ + + /* per-test cleanup + * well-behaved, even if `purl` is NULL + */ + FAIL_IF( FAIL == H5FD_s3comms_free_purl(purl) ) + purl = NULL; + + } /* for each testcase */ + + PASSED(); + return 0; + +error: + /*********** + * cleanup * + ***********/ + (void)H5FD_s3comms_free_purl(purl); + + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_parse_url */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_percent_encode_char() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_percent_encode_char()` + * + * Return: + * + * Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_percent_encode_char(void) +{ + /************************* + * test-local structures * + *************************/ + +#ifdef H5_HAVE_ROS3_VFD + struct testcase { + const char c; + const char *exp; + size_t exp_len; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + {'$', "%24", 3}, /* u+0024 dollar sign */ + {' ', "%20", 3}, /* u+0020 space */ + {'^', "%5E", 3}, /* u+0094 carat */ + {'/', "%2F", 3}, /* u+002f solidus (forward slash) */ + /* {??, "%C5%8C", 6},*/ /* u+014c Latin Capital Letter O with Macron */ + /* Not included because it is multibyte "wide" character that poses */ + /* issues both in the underlying function and in being written in */ + /* this file. */ + /* {'¢', "%C2%A2", 6}, */ /* u+00a2 cent sign */ + /* above works, but complains about wide character overflow */ + /* Elide for now, until it is determined (a) unnecessary or */ + /* (b) requiring signature change to accommodate wide characters */ + {'\0', "%00", 3}, /* u+0000 null */ + }; + char dest[13]; + size_t dest_len = 0; + int i = 0; + int n_cases = 5; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("percent encode characters"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + for (i = 0; i < n_cases; i++) { + JSVERIFY( SUCCEED, + H5FD_s3comms_percent_encode_char( + dest, + (const unsigned char)cases[i].c, + &dest_len), + NULL ) + JSVERIFY(cases[i].exp_len, dest_len, NULL ) + JSVERIFY(0, strncmp(dest, cases[i].exp, dest_len), NULL ) + JSVERIFY_STR( cases[i].exp, dest, NULL ) + } + + JSVERIFY( FAIL, + H5FD_s3comms_percent_encode_char( + NULL, + (const unsigned char)'^', + &dest_len), + NULL ) + + PASSED(); + return 0; + +error: + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_percent_encode_char */ + + +/*--------------------------------------------------------------------------- + * Function: test_s3r_open() + * + * Programmer: Jacob Smith 2018-01-24 + * + * Changes: None + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_s3r_get_filesize(void) +{ +#ifdef H5_HAVE_ROS3_VFD + + /************************ + * test-local variables * + ************************/ + + char url_raven[S3_TEST_MAX_URL_SIZE]; + s3r_t *handle = NULL; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("s3r_get_filesize"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + /* setup -- compose url to target resource + */ + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + JSVERIFY( 0, H5FD_s3comms_s3r_get_filesize(NULL), + "filesize of the null handle should be 0" ) + + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + FAIL_IF( handle == NULL ) + + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + + + FAIL_IF( SUCCEED != H5FD_s3comms_s3r_close(handle) ) + + PASSED(); + return 0; + +error: + if (handle != NULL) + (void)H5FD_s3comms_s3r_close(handle); + + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_s3r_get_filesize */ + + +/*--------------------------------------------------------------------------- + * Function: test_s3r_open() + * + * Programmer: Jacob Smith 2018-01-?? + * + * Changes: None + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_s3r_open(void) +{ +#ifdef H5_HAVE_ROS3_VFD + + /************************ + * test-local variables * + ************************/ + + char url_missing[S3_TEST_MAX_URL_SIZE]; + char url_raven[S3_TEST_MAX_URL_SIZE]; + char url_raven_badport[S3_TEST_MAX_URL_SIZE]; + char url_shakespeare[S3_TEST_MAX_URL_SIZE]; + unsigned char signing_key[SHA256_DIGEST_LENGTH]; + struct tm *now = NULL; + char iso8601now[ISO8601_SIZE]; + s3r_t *handle = NULL; + hbool_t curl_ready = FALSE; + parsed_url_t *purl = NULL; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("s3r_open"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /****************** + * PRE-TEST SETUP * + ******************/ + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_shakespeare, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_RESTRICTED) ); + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_missing, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_MISSING) ); + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + /* Set given bucket url with invalid/inactive port number for badport. + * Note, this sort of micro-management of parsed_url_t is not advised + */ + FAIL_IF( FAIL == H5FD_s3comms_parse_url(s3_test_bucket_url, &purl) ) + if (purl->port == NULL) { + purl->port = (char *)H5MM_malloc(sizeof(char) * 5); + FAIL_IF( purl->port == NULL ); + FAIL_IF( 5 < HDsnprintf(purl->port, 5, "9000") ) + } else if (strcmp(purl->port, "9000") != 0) { + FAIL_IF( 5 < HDsnprintf(purl->port, 5, "9000") ) + } else { + FAIL_IF( 5 < HDsnprintf(purl->port, 5, "1234") ) + } + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven_badport, + S3_TEST_MAX_URL_SIZE, + "%s://%s:%s/%s", + purl->scheme, + purl->host, + purl->port, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + curl_global_init(CURL_GLOBAL_DEFAULT); + curl_ready = TRUE; + + now = gmnow(); + FAIL_IF( now == NULL ) + FAIL_IF( ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1) ); + + /* It is desired to have means available to verify that signing_key + * was set successfully and to an expected value. + */ + FAIL_IF( FAIL == + H5FD_s3comms_signing_key( + signing_key, + (const char *)s3_test_aws_secret_access_key, + (const char *)s3_test_aws_region, + (const char *)iso8601now) ); + + /************************* + * OPEN NONEXISTENT FILE * + *************************/ + + /* attempt anonymously + */ + handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL); + FAIL_IF( handle != NULL ); + + /* attempt with authentication + */ + handle = H5FD_s3comms_s3r_open( + url_missing, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key); + FAIL_IF( handle != NULL ); + + /************************* + * INACTIVE PORT ON HOST * + *************************/ + +#if S3_TEST_RUN_TIMEOUT +printf("Opening on inactive port may hang for a minute; waiting for timeout\n"); + handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL); + FAIL_IF( handle != NULL ); +#endif + + /******************************* + * INVALID AUTHENTICATION INFO * + *******************************/ + + /* anonymous access on restricted file + */ + handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL); + FAIL_IF( handle != NULL ); + + /* passed in a bad ID + */ + handle = H5FD_s3comms_s3r_open( + url_shakespeare, + (const char *)s3_test_aws_region, + "I_MADE_UP_MY_ID", + (const unsigned char *)signing_key); + FAIL_IF( handle != NULL ); + + /* using an invalid signing key + */ + handle = H5FD_s3comms_s3r_open( + url_shakespeare, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)EMPTY_SHA256); + FAIL_IF( handle != NULL ); + + /******************************* + * SUCCESSFUL OPEN (AND CLOSE) * + *******************************/ + + /* anonymous + */ + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + FAIL_IF( handle == NULL ); + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), + "did not get expected filesize" ) + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + /* using authentication on anonymously-accessible file? + */ + handle = H5FD_s3comms_s3r_open( + url_raven, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key); + FAIL_IF( handle == NULL ); + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + /* authenticating + */ + handle = H5FD_s3comms_s3r_open( + url_shakespeare, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key); + FAIL_IF( handle == NULL ); + JSVERIFY( 5458199, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + + + curl_global_cleanup(); + curl_ready = FALSE; + + FAIL_IF( FAIL == H5FD_s3comms_free_purl(purl) ) + purl = NULL; + + PASSED(); + return 0; +error: + /*********** + * cleanup * + ***********/ + + if (handle != NULL) + H5FD_s3comms_s3r_close(handle); + if (purl != NULL) + H5FD_s3comms_free_purl(purl); + if (curl_ready == TRUE) + curl_global_cleanup(); + + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_s3r_open */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_s3r_read() + * + * Purpose: + * + * Specify and demonstrate the use and life cycle of an S3 Request handle + * `s3r_t`, through its related functions. + * + * H5FD_s3comms_s3r_open + * H5FD_s3comms_s3r_getsize << called by open() _only_ + * H5FD_s3comms_s3r_read << called by getsize(), multiple times working + * H5FD_s3comms_s3r_close + * + * Shows most basic curl interation. + * + * Programmer: Jacob Smith + * 2017-10-06 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_s3r_read(void) +{ +#ifdef H5_HAVE_ROS3_VFD + +#define S3COMMS_TEST_BUFFER_SIZE 256 + + /************************ + * test-local variables * + ************************/ + + char url_raven[S3_TEST_MAX_URL_SIZE]; + char buffer[S3COMMS_TEST_BUFFER_SIZE]; + s3r_t *handle = NULL; + hbool_t curl_ready = FALSE; + unsigned int i = 0; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("test_s3r_read"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + /* + * initial setup + */ + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + curl_global_init(CURL_GLOBAL_DEFAULT); + curl_ready = TRUE; + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /* open file + */ + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + FAIL_IF( handle == NULL ) + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /********************** + * read start of file * + **********************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)0, + (size_t)118, + buffer), + NULL ) + JSVERIFY_STR ( + "Once upon a midnight dreary, while I pondered, weak and weary,\n" \ + "Over many a quaint and curious volume of forgotten lore", + buffer, + NULL ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /************************ + * read arbitrary range * + ************************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)2540, + (size_t)54, + buffer), + NULL ) + JSVERIFY_STR( "the grave and stern decorum of the countenance it wore", + buffer, + NULL ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /********************** + * read one character * + **********************/ + + JSVERIFY(SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)2540, + (size_t)1, + buffer), + NULL ) + JSVERIFY_STR( "t", buffer, NULL ) + + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /*************** + * read to EoF * + ***************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)6370, + (size_t)0, + buffer), + NULL ) + JSVERIFY( 0, + strncmp(buffer, + "And my soul from out that shadow that lies floating on the floor\nShall be lifted—nevermore!\n", + 94), + buffer ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /***************** + * read past eof * + *****************/ + + JSVERIFY( FAIL, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)6400, + (size_t)100, /* 6400+100 > 6464 */ + buffer), + NULL ) + JSVERIFY( 0, strcmp("", buffer), NULL ) + + /************************ + * read starts past eof * + ************************/ + + JSVERIFY( FAIL, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)1200699, /* 1200699 > 6464 */ + (size_t)100, + buffer), + NULL ) + JSVERIFY( 0, strcmp("", buffer), NULL ) + + /********************** + * read starts on eof * + **********************/ + + JSVERIFY( FAIL, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)6464, + (size_t)0, + buffer), + NULL ) + JSVERIFY( 0, strcmp("", buffer), NULL ) + + /************* + * TEAR DOWN * + *************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * cleanup * + ***********/ + + if (handle != NULL) + H5FD_s3comms_s3r_close(handle); + + if (curl_ready == TRUE) + curl_global_cleanup(); + + return -1; + +#undef S3COMMS_TEST_BUFFER_SIZE +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_s3r_read */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_signing_key() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_signing_key()` + * + * More test cases would be a very good idea. + * + * Programmer: Jacob Smith + * 2017-09-18 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_signing_key(void) +{ +#ifdef H5_HAVE_ROS3_VFD + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *region; + const char *secret_key; + const char *when; + unsigned char exp[SHA256_DIGEST_LENGTH]; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "us-east-1", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "20130524T000000Z", + { 0xdb, 0xb8, 0x93, 0xac, 0xc0, 0x10, 0x96, 0x49, + 0x18, 0xf1, 0xfd, 0x43, 0x3a, 0xdd, 0x87, 0xc7, + 0x0e, 0x8b, 0x0d, 0xb6, 0xbe, 0x30, 0xc1, 0xfb, + 0xea, 0xfe, 0xfa, 0x5e, 0xc6, 0xba, 0x83, 0x78, + }, + }, + }; + int i = 0; + unsigned char *key = NULL; + int ncases = 1; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("signing_key"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + for (i = 0; i < ncases; i++) { + key = (unsigned char *)HDmalloc(sizeof(unsigned char) * \ + SHA256_DIGEST_LENGTH); + HDassert(key != NULL); + + JSVERIFY( SUCCEED, + H5FD_s3comms_signing_key( + key, + cases[i].secret_key, + cases[i].region, + cases[i].when), + NULL ) + + JSVERIFY( 0, + strncmp((const char *)cases[i].exp, + (const char *)key, + SHA256_DIGEST_LENGTH), + cases[i].exp ) + + free(key); + key = NULL; + } + + + /*************** + * ERROR CASES * + ***************/ + + key = (unsigned char *)HDmalloc(sizeof(unsigned char) * \ + SHA256_DIGEST_LENGTH); + HDassert(key != NULL); + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + NULL, + cases[0].secret_key, + cases[0].region, + cases[0].when), + "destination cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + key, + NULL, + cases[0].region, + cases[0].when), + "secret key cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + key, + cases[0].secret_key, + NULL, + cases[0].when), + "aws region cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + key, + cases[0].secret_key, + cases[0].region, + NULL), + "time string cannot be NULL" ) + + free(key); + key = NULL; + + PASSED(); + return 0; + +error: + if (key != NULL) { + free(key); + } + + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_signing_key */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_tostringtosign() + * + * Purpose: + * + * Verify that we can get the "string to sign" from a Canonical Request and + * related information. + * + * Demonstrate failure cases. + * + * Return: + * + * Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2017-09-13 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_tostringtosign(void) +{ +#ifdef H5_HAVE_ROS3_VFD + /************************ + * test-local variables * + ************************/ + + const char canonreq[] = "GET\n/test.txt\n\nhost:examplebucket.s3.amazonaws.com\nrange:bytes=0-9\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:20130524T000000Z\n\nhost;range;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + const char iso8601now[] = "20130524T000000Z"; + const char region[] = "us-east-1"; + char s2s[512]; +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("s3comms tostringtosign"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + puts(" ROS3 VFD is not enabled"); + fflush(stdout); + return 0; +#else + JSVERIFY( SUCCEED, + H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, region), + "unable to create string to sign" ) + + JSVERIFY_STR( "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972", + s2s, NULL ) + + JSVERIFY( FAIL, + H5FD_s3comms_tostringtosign(s2s, NULL, iso8601now, region), + "canonical request string cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_tostringtosign(s2s, canonreq, NULL, region), + "time string cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, NULL), + "aws region cannot be NULL" ) + + PASSED(); + return 0; + +error : + return -1; +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_tostringtosign */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_trim() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_trim()`. + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_trim(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *in; + size_t in_len; + const char *exp; + size_t exp_len; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "block string", + 12, + "block string", + 12, + }, + { " \n\r \t", + 6, + "", + 0, + }, + { " \twhite b4", + 10, + "white b4", + 8, + }, + { "white after\r\n ", + 15, + "white after", + 11, + }, + { " on\nends\t", + 9, + "on\nends", + 7, + }, + }; + char dest[32]; + size_t dest_len = 0; + int i = 0; + int n_cases = 5; + char *str = NULL; + + + + TESTING("s3comms trim"); + + for (i = 0; i < n_cases; i++) { + HDassert(str == NULL); + str = (char *)HDmalloc(sizeof(char) * cases[i].in_len); + HDassert(str != NULL); + HDstrncpy(str, cases[i].in, cases[i].in_len); + + JSVERIFY( SUCCEED, + H5FD_s3comms_trim(dest, str, cases[i].in_len, &dest_len), + NULL ) + JSVERIFY( cases[i].exp_len, dest_len, cases[i].in ) + if (dest_len > 0) { + JSVERIFY( 0, strncmp(cases[i].exp, dest, dest_len), + cases[i].exp ) + } + free(str); + str = NULL; + } + + JSVERIFY( SUCCEED, H5FD_s3comms_trim(dest, NULL, 3, &dest_len), + "should not fail when trimming a null string" ); + JSVERIFY( 0, dest_len, "trimming NULL string writes 0 characters" ) + + HDassert(str == NULL); + str = (char *)HDmalloc(sizeof(char *) * 11); + HDassert(str != NULL); + memcpy(str, "some text ", 11); /* string with null terminator */ + JSVERIFY( FAIL, H5FD_s3comms_trim(NULL, str, 10, &dest_len), + "destination for trim cannot be NULL" ); + free(str); + str = NULL; + + PASSED(); + return 0; + +error: + if (str != NULL) + free(str); + return -1; + +} /* test_trim */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_uriencode() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_uriencode()`. + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_uriencode(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *str; + size_t s_len; + hbool_t encode_slash; + const char *expected; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "/path/to/resource.jpg", + 21, + FALSE, + "/path/to/resource.jpg", + }, + { "/path/to/resource.jpg", + 21, + TRUE, + "%2Fpath%2Fto%2Fresource.jpg", + }, + { "string got_spaa ces", + 20, + TRUE, + "string%20got_spaa%20%20ces", + }, + { "sp ac~es/and-sl ash.encoded", + 27, + TRUE, + "sp%20ac~es%2Fand-sl%20ash.encoded", + }, + { "sp ac~es/and-sl ash.unencoded", + 29, + FALSE, + "sp%20ac~es/and-sl%20ash.unencoded", + }, + { "/path/to/resource.txt", + 0, + FALSE, + "", + + } + }; + char *dest = NULL; + size_t dest_written = 0; + int i = 0; + int ncases = 6; + size_t str_len = 0; + + + + TESTING("s3comms uriencode") + + for (i = 0; i < ncases; i++) { + str_len = cases[i].s_len; + dest = (char *)HDmalloc(sizeof(char) * str_len * 3 + 1); + FAIL_IF( dest == NULL ) + + JSVERIFY( SUCCEED, + H5FD_s3comms_uriencode( + dest, + cases[i].str, + str_len, + cases[i].encode_slash, + &dest_written), + NULL ); + JSVERIFY( HDstrlen(cases[i].expected), + dest_written, + NULL ) + JSVERIFY( 0, + strncmp(dest, cases[i].expected, dest_written), + cases[i].expected ); + + free(dest); + dest = NULL; + } + + /*************** + * ERROR CASES * + ***************/ + + dest = (char *)HDmalloc(sizeof(char) * 15); + HDassert(dest != NULL); + + JSVERIFY( FAIL, + H5FD_s3comms_uriencode(NULL, "word$", 5, false, &dest_written), + "destination cannot be NULL" ); + JSVERIFY( FAIL, + H5FD_s3comms_uriencode(dest, NULL, 5, false, &dest_written), + "source string cannot be NULL" ); + + free(dest); + dest = NULL; + + PASSED(); + return 0; + +error: + if (dest != NULL) { + free(dest); + } + return -1; + +} /* test_uriencode */ + + + + +/*------------------------------------------------------------------------- + * Function: main() + * + * Purpose: + * + * Run unit tests for S3 Communications (s3comms). + * + * Return: + * + * Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2017-10-12 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; +#ifdef H5_HAVE_ROS3_VFD + const char *bucket_url_env = NULL; +#endif /* H5_HAVE_ROS3_VFD */ + + h5_reset(); + + HDprintf("Testing S3Communications functionality.\n"); + +#ifdef H5_HAVE_ROS3_VFD + + /* "clear" profile data strings */ + s3_test_aws_access_key_id[0] = '\0'; + s3_test_aws_secret_access_key[0] = '\0'; + s3_test_aws_region[0] = '\0'; + s3_test_bucket_url[0] = '\0'; + +/* TODO: unit/regression test for H5FD_s3comms_load_aws_profile() + * requires a few test files and/or manipulation of default path + */ + /* attempt to load test credentials + * if unable, certain tests will be skipped + */ + if (SUCCEED == H5FD_s3comms_load_aws_profile( + S3_TEST_PROFILE_NAME, + s3_test_aws_access_key_id, + s3_test_aws_secret_access_key, + s3_test_aws_region)) + { + s3_test_credentials_loaded = 1; + } + + bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL"); + if (bucket_url_env == NULL || bucket_url_env[0] == '\0') { + HDprintf("WARNING: S3 bucket url is not defined in enviornment " \ + "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n"); + } else { + HDstrncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE); + s3_test_bucket_defined = TRUE; + } + +#endif /* H5_HAVE_ROS3_VFD */ + + /* tests ordered rougly by dependence */ + nerrors += test_macro_format_credential() < 0 ? 1 : 0; + nerrors += test_trim() < 0 ? 1 : 0; + nerrors += test_nlowercase() < 0 ? 1 : 0; + nerrors += test_uriencode() < 0 ? 1 : 0; + nerrors += test_percent_encode_char() < 0 ? 1 : 0; + nerrors += test_bytes_to_hex() < 0 ? 1 : 0; + nerrors += test_HMAC_SHA256() < 0 ? 1 : 0; + nerrors += test_signing_key() < 0 ? 1 : 0; + nerrors += test_hrb_node_set() < 0 ? 1 : 0; + nerrors += test_hrb_init_request() < 0 ? 1 : 0; + nerrors += test_parse_url() < 0 ? 1 : 0; + nerrors += test_aws_canonical_request() < 0 ? 1 : 0; + nerrors += test_tostringtosign() < 0 ? 1 : 0; + nerrors += test_s3r_open() < 0 ? 1 : 0; + nerrors += test_s3r_get_filesize() < 0 ? 1 : 0; + nerrors += test_s3r_read() < 0 ? 1 : 0; + + if(nerrors) { + HDprintf("***** %d S3comms TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + return 1; + } /* end if */ + + HDprintf("All S3comms tests passed.\n"); + + return 0; +} /* end main() */ + @@ -58,6 +58,7 @@ const char *FILENAME[] = { "stdio_file", /*7*/ "windows_file", /*8*/ "new_multi_file_v16",/*9*/ + "ro_s3_file6", /*10*/ NULL }; @@ -66,7 +67,7 @@ const char *FILENAME[] = { #define COMPAT_BASENAME "family_v16_" #define MULTI_COMPAT_BASENAME "multi_file_v16" - + /*------------------------------------------------------------------------- * Function: test_sec2 * @@ -178,7 +179,7 @@ error: return -1; } /* end test_sec2() */ - + /*------------------------------------------------------------------------- * Function: test_core * @@ -534,7 +535,7 @@ error: return -1; } /* end test_core() */ - + /*------------------------------------------------------------------------- * Function: test_direct * @@ -754,7 +755,7 @@ error: #endif /*H5_HAVE_DIRECT*/ } - + /*------------------------------------------------------------------------- * Function: test_family_opens * @@ -835,7 +836,7 @@ error: } /* end test_family_opens() */ #pragma GCC diagnostic pop - + /*------------------------------------------------------------------------- * Function: test_family * @@ -1017,7 +1018,7 @@ error: return -1; } - + /*------------------------------------------------------------------------- * Function: test_family_compat * @@ -1129,7 +1130,7 @@ error: } /* end test_family_compat() */ #pragma GCC diagnostic pop - + /*------------------------------------------------------------------------- * Function: test_multi_opens * @@ -1170,7 +1171,7 @@ test_multi_opens(char *fname) } /* end test_multi_opens() */ #pragma GCC diagnostic pop - + /*------------------------------------------------------------------------- * Function: test_multi * @@ -1404,7 +1405,7 @@ error: return FAIL; } /* end test_multi() */ - + /*------------------------------------------------------------------------- * Function: test_multi_compat * @@ -1578,7 +1579,7 @@ error: return -1; } - + /*------------------------------------------------------------------------- * Function: test_log * @@ -1689,7 +1690,7 @@ error: return -1; } - + /*------------------------------------------------------------------------- * Function: test_stdio * @@ -1794,7 +1795,7 @@ error: } - + /*------------------------------------------------------------------------- * Function: test_windows * @@ -1916,7 +1917,146 @@ error: } /* end test_windows() */ - +/*------------------------------------------------------------------------- + * Function: test_ros3 + * + * Purpose: Tests the file handle interface for the ROS3 driver + * + * As the ROS3 driver is 1) read only, 2) requires access + * to an S3 server (minio for now), this test is quite + * different from the other tests. + * + * For now, test only fapl & flags. Extend as the + * work on the VFD continues. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: John Mainzer + * 7/12/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_ros3(void) +{ + hid_t fid = -1; /* file ID */ + hid_t fapl_id = -1; /* file access property list ID */ + hid_t fapl_id_out = -1; /* from H5Fget_access_plist */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + char filename[1024]; /* filename */ + void *os_file_handle = NULL; /* OS file handle */ + hsize_t file_size; /* file size */ + H5FD_ros3_fapl_t test_ros3_fa; + H5FD_ros3_fapl_t ros3_fa_0 = + { + /* version = */ H5FD__CURR_ROS3_FAPL_T_VERSION, + /* authenticate = */ FALSE, + /* aws_region = */ "", + /* secret_id = */ "", + /* secret_key = */ "plugh", + }; + + TESTING("ROS3 file driver"); + + /* Set property list and file name for ROS3 driver. */ + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if(H5Pset_fapl_ros3(fapl_id, &ros3_fa_0) < 0) + TEST_ERROR; + + /* verify that the ROS3 FAPL entry is set as expected */ + if(H5Pget_fapl_ros3(fapl_id, &test_ros3_fa) < 0) + TEST_ERROR; + + /* need a macro to compare instances of H5FD_ros3_fapl_t */ + if((test_ros3_fa.version != ros3_fa_0.version) || + (test_ros3_fa.authenticate != ros3_fa_0.authenticate) || + (strcmp(test_ros3_fa.aws_region, ros3_fa_0.aws_region) != 0) || + (strcmp(test_ros3_fa.secret_id, ros3_fa_0.secret_id) != 0) || + (strcmp(test_ros3_fa.secret_key, ros3_fa_0.secret_key) != 0)) + TEST_ERROR; + + h5_fixname(FILENAME[10], fapl_id, filename, sizeof(filename)); + + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR; + + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR; + + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_DATA_SIEVE)) + TEST_ERROR + + /* can't create analogs of the following tests until the + * ROS3 driver is up and running in a minimal fashion. + * Comment them out until we get to them. + */ +#if 0 + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + TEST_ERROR; + + /* Retrieve the access property list... */ + if((fapl_id_out = H5Fget_access_plist(fid)) < 0) + TEST_ERROR; + + /* Check that the driver is correct */ + if(H5FD_ROS3 != H5Pget_driver(fapl_id_out)) + TEST_ERROR; + + /* ...and close the property list */ + if(H5Pclose(fapl_id_out) < 0) + TEST_ERROR; + + /* Check that we can get an operating-system-specific handle from + * the library. + */ + if(H5Fget_vfd_handle(fid, H5P_DEFAULT, &os_file_handle) < 0) + TEST_ERROR; + if(os_file_handle == NULL) + FAIL_PUTS_ERROR("NULL os-specific vfd/file handle was returned from H5Fget_vfd_handle"); + + + /* There is no garantee the size of metadata in file is constant. + * Just try to check if it's reasonable. + * + * Currently it should be around 2 KB. + */ + if(H5Fget_filesize(fid, &file_size) < 0) + TEST_ERROR; + if(file_size < 1 * KB || file_size > 4 * KB) + FAIL_PUTS_ERROR("suspicious file size obtained from H5Fget_filesize"); + + /* Close and delete the file */ + if(H5Fclose(fid) < 0) + TEST_ERROR; + h5_delete_test_file(FILENAME[0], fapl_id); + + /* Close the fapl */ + if(H5Pclose(fapl_id) < 0) + TEST_ERROR; +#endif + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl_id); + H5Pclose(fapl_id_out); + H5Fclose(fid); + } H5E_END_TRY; + return -1; +} /* end test_ros3() */ + + + /*------------------------------------------------------------------------- * Function: main * @@ -1949,6 +2089,7 @@ main(void) nerrors += test_log() < 0 ? 1 : 0; nerrors += test_stdio() < 0 ? 1 : 0; nerrors += test_windows() < 0 ? 1 : 0; + nerrors += test_ros3() < 0 ? 1 : 0; if(nerrors) { HDprintf("***** %d Virtual File Driver TEST%s FAILED! *****\n", |