diff options
author | Jake Smith <jake.smith@hdfgroup.org> | 2019-08-14 20:54:10 (GMT) |
---|---|---|
committer | Jake Smith <jake.smith@hdfgroup.org> | 2019-08-14 20:54:10 (GMT) |
commit | 396edbe6bf053b9acc432b45eb47c3bfbdbbefed (patch) | |
tree | 1b1638303a96d5cf19622498cd10634aa357e462 | |
parent | 47e3d99f65b5f610c3de61c3e9060b6b30b71ca0 (diff) | |
parent | 2e5cd3d2b2d953420252c80ab5e32ac95ac0c5b3 (diff) | |
download | hdf5-396edbe6bf053b9acc432b45eb47c3bfbdbbefed.zip hdf5-396edbe6bf053b9acc432b45eb47c3bfbdbbefed.tar.gz hdf5-396edbe6bf053b9acc432b45eb47c3bfbdbbefed.tar.bz2 |
Merge pull request #1872 in HDFFV/hdf5 from ~JAKE.SMITH/hdf5:bugfix/ros3_hdfs_warns_squashed to develop
* commit '2e5cd3d2b2d953420252c80ab5e32ac95ac0c5b3':
Fix compiler warnings with diabled ROS3 and HDFS VFDs. Fix misuse of [HD]strncpy and [HD]snprintf. Minor formatting changes. Fix test cleanup issue with java groups example.
-rw-r--r-- | java/examples/groups/JavaGroupExample.sh.in | 14 | ||||
-rw-r--r-- | src/H5FDhdfs.c | 484 | ||||
-rw-r--r-- | src/H5FDros3.c | 392 | ||||
-rw-r--r-- | src/H5FDs3comms.c | 863 | ||||
-rw-r--r-- | src/H5FDs3comms.h | 2 | ||||
-rw-r--r-- | test/hdfs.c | 443 | ||||
-rw-r--r-- | test/ros3.c | 170 | ||||
-rw-r--r-- | test/s3comms.c | 215 | ||||
-rw-r--r-- | tools/src/h5dump/h5dump.c | 4 | ||||
-rw-r--r-- | tools/src/h5ls/h5ls.c | 4 | ||||
-rw-r--r-- | tools/src/h5stat/h5stat.c | 8 |
11 files changed, 1416 insertions, 1183 deletions
diff --git a/java/examples/groups/JavaGroupExample.sh.in b/java/examples/groups/JavaGroupExample.sh.in index 9d4673f..acc606f 100644 --- a/java/examples/groups/JavaGroupExample.sh.in +++ b/java/examples/groups/JavaGroupExample.sh.in @@ -179,17 +179,9 @@ COPY_DATAFILES_TO_BLDDIR() CLEAN_DATAFILES_AND_BLDDIR() { - $RM $BLDDIR/examples.groups.H5Ex_G_*.txt - $RM $BLDDIR/H5Ex_G_*.out - $RM $BLDDIR/H5Ex_G_*.h5 - $RM $BLDDIR/h5ex_g_*.h5 - SDIR=`$DIRNAME $tstfile` - INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` - INODE_DDIR=`$LS -i -d $BLDDIR | $AWK -F' ' '{print $1}'` - if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then SDIR=`$DIRNAME $tstfile` - $RM $BLDDIR/h5ex_g_iterate.h5 - $RM $BLDDIR/h5ex_g_visit.h5 - fi + $RM $BLDDIR/examples.groups.H5Ex_G_*.txt + $RM $BLDDIR/H5Ex_G_*.out + $RM $BLDDIR/H5Ex_G_*.h5 } COPY_ITERFILES="$LIST_ITER_FILES" diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c index e3e11b2..83d0202 100644 --- a/src/H5FDhdfs.c +++ b/src/H5FDhdfs.c @@ -130,6 +130,8 @@ typedef struct { #endif /* HDFS_STATS */ +#ifdef H5_HAVE_LIBHDFS + /* "unique" identifier for `hdfs_t` structures. * Randomly generated by unweighted dice rolls. */ @@ -143,7 +145,7 @@ typedef struct { * Purpose: * * Contain/retain information associated with a file hosted on Hadoop - * Distributed File System (HDFS). Instantiated and populated via + * Distributed File System (HDFS). Instantiated and populated via * `H5FD_hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`. * * @@ -179,14 +181,11 @@ typedef struct { */ typedef struct { unsigned long magic; -#ifdef H5_HAVE_LIBHDFS hdfsFS filesystem; hdfsFileInfo *fileinfo; - hdfsFile file; -#endif + hdfsFile file; } hdfs_t; -#ifdef H5_HAVE_LIBHDFS /*-------------------------------------------------------------------------- * Function: H5FD_hdfs_handle_open @@ -274,7 +273,7 @@ H5FD_hdfs_handle_open( "hdfsGetPathInfo failed") } handle->file = hdfsOpenFile( - handle->filesystem, + handle->filesystem, path, O_RDONLY, stream_buffer_size, @@ -292,18 +291,18 @@ done: /* error; clean up */ HDassert(handle->magic == HDFS_HDFST_MAGIC); handle->magic++; - if (handle->file != NULL) { + if (handle->file != NULL) { if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close hdfs file handle") } } - if (handle->fileinfo != NULL) { + if (handle->fileinfo != NULL) { hdfsFreeFileInfo(handle->fileinfo, 1); } - if (handle->filesystem != NULL) { + if (handle->filesystem != NULL) { if (FAIL == (hdfsDisconnect(handle->filesystem))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to disconnect from hdfs") } } @@ -351,18 +350,18 @@ H5FD_hdfs_handle_close(hdfs_t *handle) } handle->magic++; - if (handle->file != NULL) { + if (handle->file != NULL) { if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close hdfs file handle") } } - if (handle->fileinfo != NULL) { + if (handle->fileinfo != NULL) { hdfsFreeFileInfo(handle->fileinfo, 1); } - if (handle->filesystem != NULL) { + if (handle->filesystem != NULL) { if (FAIL == (hdfsDisconnect(handle->filesystem))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to disconnect hdfs file system") } } @@ -373,6 +372,7 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_close() */ + #endif /* H5_HAVE_LIBHDFS */ @@ -382,9 +382,9 @@ done: * * Purpose: * - * H5FD_hdfs_t is a structure used to store all information needed to + * H5FD_hdfs_t is a structure used to store all information needed to * maintain R/O access to a single HDF5 file in an HDFS file system. - * This structure is created when such a file is "opened" and + * This structure is created when such a file is "opened" and * discarded when it is "closed". * * @@ -396,7 +396,7 @@ done: * * `fa` (H5FD_hdfs_fapl_t) * - * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data + * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data * needed to "open" the HDF5 file. * * `eoa` (haddr_t) @@ -405,9 +405,9 @@ done: * equal the file size. * * `hdfs_handle` (hdfs_t *) - * + * * Instance of HDFS Request handle associated with the target resource. - * Responsible for communicating with remote host and presenting file + * Responsible for communicating with remote host and presenting file * contents as indistinguishable from a file on the local filesystem. * * *** present only if HDFS_SATS is flagged to enable stats collection *** @@ -456,10 +456,13 @@ typedef struct H5FD_hdfs_t { * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' * is too large to be represented by the second argument * of the file seek function. + * Only included if HDFS code should compile. * */ #define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#ifdef H5_HAVE_LIBHDFS #define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#endif /* H5_HAVE_LIBHDFS */ /* Prototypes */ static herr_t H5FD_hdfs_term(void); @@ -474,13 +477,13 @@ static herr_t H5FD_hdfs_query(const H5FD_t *_f1, unsigned long *flags); static haddr_t H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t type); static herr_t H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); static haddr_t H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, +static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); -static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, void *buf); -static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, const void *buf); -static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, +static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD_hdfs_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD_hdfs_unlock(H5FD_t *_file); @@ -521,8 +524,10 @@ static const H5FD_class_t H5FD_hdfs_g = { H5FD_FLMAP_DICHOTOMY /* fl_map */ }; +#ifdef H5_HAVE_LIBHDFS /* Declare a free list to manage the H5FD_hdfs_t struct */ H5FL_DEFINE_STATIC(H5FD_hdfs_t); +#endif /* H5_HAVE_LIBHDFS */ /*------------------------------------------------------------------------- @@ -545,7 +550,7 @@ H5FD__init_package(void) FUNC_ENTER_STATIC if (H5FD_hdfs_init() < 0) { - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize hdfs VFD") } @@ -643,8 +648,8 @@ H5FD_hdfs_term(void) * Function: H5Pset_fapl_hdfs * * Purpose: Modify the file access property list to use the H5FD_HDFS - * driver defined in this source file. All driver specfic - * properties are passed in as a pointer to a suitably + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably * initialized instance of H5FD_hdfs_fapl_t * * Return: SUCCEED/FAIL @@ -658,7 +663,7 @@ H5FD_hdfs_term(void) *------------------------------------------------------------------------- */ herr_t -H5Pset_fapl_hdfs(hid_t fapl_id, +H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa) { H5P_genplist_t *plist = NULL; /* Property list pointer */ @@ -674,7 +679,7 @@ H5Pset_fapl_hdfs(hid_t fapl_id, #endif plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); - if (plist == NULL) { + if (plist == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, \ "not a file access property list") } @@ -699,12 +704,12 @@ done: * contains internally consistant data. Return SUCCEED if so, * and FAIL otherwise. * - * Note the difference between internally consistant and - * correct. As we will have to try to access the target + * Note the difference between internally consistant and + * correct. As we will have to try to access the target * object to determine whether the supplied data is correct, * we will settle for internal consistancy at this point * - * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally + * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally * consistant data, FAIL otherwise. * * Programmer: Jacob Smith @@ -761,7 +766,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_fapl_hdfs(hid_t fapl_id, +H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out) { const H5FD_hdfs_fapl_t *fa = NULL; @@ -832,7 +837,7 @@ H5FD_hdfs_fapl_get(H5FD_t *_file) fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t)); if (fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") } @@ -878,7 +883,7 @@ H5FD_hdfs_fapl_copy(const void *_old_fa) new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t)); if (new_fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") } @@ -916,7 +921,7 @@ H5FD_hdfs_fapl_free(void *_fa) FUNC_ENTER_NOAPI_NOINIT_NOERR - HDassert(fa != NULL); /* sanity check */ + HDassert(fa != NULL); /* sanity check */ H5MM_xfree(fa); @@ -994,11 +999,11 @@ done: * Create and/or opens a file as an HDF5 file. * * Any flag except H5F_ACC_RDONLY will cause an error. - * + * * Return: * - * Success: A pointer to a new file data structure. - * The public fields will be initialized by the caller, which is + * Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is * always H5FD_open(). * * Failure: NULL @@ -1010,26 +1015,21 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS static H5FD_t * H5FD_hdfs_open( - const char *path, - unsigned flags, - hid_t fapl_id, + const char *path, + unsigned flags, + hid_t fapl_id, haddr_t maxaddr) { H5FD_t *ret_value = NULL; -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = NULL; hdfs_t *handle = NULL; H5FD_hdfs_fapl_t fa; -#endif FUNC_ENTER_NOAPI_NOINIT -#ifndef H5_HAVE_LIBHDFS - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, - "Illegal open of unsupported virtual file (hdfs)"); -#else #if HDFS_DEBUG HDfprintf(stdout, "H5FD_hdfs_open() called.\n"); #endif /* HDFS_DEBUG */ @@ -1078,11 +1078,11 @@ H5FD_hdfs_open( HDassert(handle->magic == HDFS_HDFST_MAGIC); - /* create new file struct + /* create new file struct */ file = H5FL_CALLOC(H5FD_hdfs_t); if (file == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") } file->hdfs_handle = handle; @@ -1090,33 +1090,53 @@ H5FD_hdfs_open( #if HDFS_STATS if (FAIL == hdfs_reset_stats(file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics") } #endif /* HDFS_STATS */ ret_value = (H5FD_t*)file; -#endif /* H5_HAVE_LIBHDFS */ done: -#ifdef H5_HAVE_LIBHDFS if (ret_value == NULL) { - if (handle != NULL) { + if (handle != NULL) { if (FAIL == H5FD_hdfs_handle_close(handle)) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close HDFS file handle") } } if (file != NULL) { file = H5FL_FREE(H5FD_hdfs_t, file); } - } /* if null return value (error) */ -#endif /* H5_HAVE_LIBHDFS */ + } /* end if null return value (error) */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_open() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static H5FD_t * +H5FD_hdfs_open( + const char H5_ATTR_UNUSED *path, + unsigned H5_ATTR_UNUSED flags, + hid_t H5_ATTR_UNUSED fapl_id, + haddr_t H5_ATTR_UNUSED maxaddr) +{ + H5FD_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, + "Illegal open of unsupported virtual file (hdfs)"); +done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_open() */ +#endif /* H5_HAVE_LIBHDFS */ + #if HDFS_STATS /*---------------------------------------------------------------------------- @@ -1124,7 +1144,7 @@ done: * Function: hdfs_fprint_stats() * * Purpose: - * + * * Tabulate and pretty-print statistics for this virtual file. * * Should be called upon file close. @@ -1145,17 +1165,17 @@ done: * - max (largest size read) * 2. tabulation of "bins", sepraring reads into exponentially-larger * ranges of size. - * - columns for number of reads, total bytes, and average size, with + * - columns for number of reads, total bytes, and average size, with * separate sub-colums for raw- and metadata reads. * - each row represents one bin, identified by the top of its range - * + * * Bin ranges can be modified with pound-defines at the top of this file. * * Bins without any reads in their bounds are not printed. * * An "overflow" bin is also present, to catch "big" reads. * - * Output for all bins (and range ceiling and average size report) + * Output for all bins (and range ceiling and average size report) * is divied by powers of 1024. By corollary, four digits before the decimal * is valid. * @@ -1216,36 +1236,6 @@ hdfs_fprint_stats( "hdfs handle has invalid magic") } - /* TODO: See what libhdfs exposes to us. */ - -#if 0 - if (file->s3r_handle->purl == NULL) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "parsed url structure cannot be null") - } - purl = file->s3r_handle->purl; - - /****************** - * PRINT FILENAME * - ******************/ - - HDfprintf(stream, "stats for %s://%s", purl->scheme, purl->host); - if (purl->port != NULL && purl->port[0] != '\0') { - HDfprintf(stream, ":%s", purl->port); - } - if (purl->query != NULL && purl->query[0] != '\0') { - if (purl->path != NULL && purl->path[0] != '\0') { - HDfprintf(stream, "/%s", purl->path); - } else { - HDfprintf(stream, "/"); - } - HDfprintf(stream, "?%s", purl->query); - } else if (purl->path != NULL && purl->path[0] != '\0') { - HDfprintf(stream, "/%s", purl->path); - } - HDfprintf(stream, "\n"); -#endif - /******************* * AGGREGATE STATS * *******************/ @@ -1254,18 +1244,22 @@ hdfs_fprint_stats( const hdfs_statsbin *r = &file->raw[i]; const hdfs_statsbin *m = &file->meta[i]; - if (m->min < min_meta) min_meta = m->min; - if (r->min < min_raw) min_raw = r->min; - if (m->max > max_meta) max_meta = m->max; - if (r->max > max_raw) max_raw = r->max; + if (m->min < min_meta) { min_meta = m->min; } + if (r->min < min_raw) { min_raw = r->min; } + if (m->max > max_meta) { max_meta = m->max; } + if (r->max > max_raw) { max_raw = r->max; } count_raw += r->count; count_meta += m->count; bytes_raw += r->bytes; bytes_meta += m->bytes; } - if (count_raw > 0) average_raw = (double)bytes_raw / (double)count_raw; - if (count_meta > 0) average_meta = (double)bytes_meta / (double)count_meta; + if (count_raw > 0) { + average_raw = (double)bytes_raw / (double)count_raw; + } + if (count_meta > 0) { + average_meta = (double)bytes_meta / (double)count_meta; + } /****************** * PRINT OVERVIEW * @@ -1342,9 +1336,9 @@ hdfs_fprint_stats( * PRINT INDIVIDUAL BIN STATS * ******************************/ - HDfprintf(stream, + HDfprintf(stream, "BINS # of reads total bytes average size\n"); - HDfprintf(stream, + HDfprintf(stream, " up-to meta raw meta raw meta raw\n"); for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { @@ -1414,7 +1408,7 @@ hdfs_fprint_stats( HDassert(suffix_i < sizeof(suffixes)); HDfprintf( - stream, + stream, " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, suffixes[suffix_i], /* bin ceiling */ m->count, /* metadata reads */ @@ -1428,7 +1422,6 @@ hdfs_fprint_stats( done: FUNC_LEAVE_NOAPI(ret_value); - } /* hdfs_fprint_stats */ #endif /* HDFS_STATS */ @@ -1442,7 +1435,7 @@ done: * Close an HDF5 file. * * Return: - * + * * SUCCEED/FAIL * * Programmer: Jacob Smith @@ -1452,35 +1445,31 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_close(H5FD_t *_file) { herr_t ret_value = SUCCEED; -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; -#endif FUNC_ENTER_NOAPI_NOINIT -#ifndef H5_HAVE_LIBHDFS - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, - "Illegal close of unsupported Virtual File (hdfs)") -#else #if HDFS_DEBUG HDfprintf(stdout, "H5FD_hdfs_close() called.\n"); #endif - /* Sanity checks + /* Sanity checks */ HDassert(file != NULL); HDassert(file->hdfs_handle != NULL); HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); - /* Close the underlying request handle + /* Close the underlying request handle */ - if (file->hdfs_handle != NULL) { + if (file->hdfs_handle != NULL) { if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) { - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close HDFS file handle") } } @@ -1488,21 +1477,37 @@ H5FD_hdfs_close(H5FD_t *_file) #if HDFS_STATS /* TODO: mechanism to re-target stats printout */ if (FAIL == hdfs_fprint_stats(stdout, file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics") } #endif /* HDFS_STATS */ - /* Release the file info + /* Release the file info */ file = H5FL_FREE(H5FD_hdfs_t, file); -#endif /* H5_HAVE_LIBHDFS */ done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_close() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "Illegal close of unsupported Virtual File (hdfs)") + +done: + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_close() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1527,21 +1532,21 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static int -H5FD_hdfs_cmp(const H5FD_t *_f1, - const H5FD_t *_f2) +H5FD_hdfs_cmp( + const H5FD_t *_f1, + const H5FD_t *_f2) { int ret_value = 0; -#ifdef H5_HAVE_LIBHDFS const H5FD_hdfs_t *f1 = (const H5FD_hdfs_t *)_f1; const H5FD_hdfs_t *f2 = (const H5FD_hdfs_t *)_f2; hdfsFileInfo *finfo1 = NULL; hdfsFileInfo *finfo2 = NULL; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR -#ifdef H5_HAVE_LIBHDFS #if HDFS_DEBUG HDfprintf(stdout, "H5FD_hdfs_cmp() called.\n"); #endif /* HDFS_DEBUG */ @@ -1556,23 +1561,37 @@ H5FD_hdfs_cmp(const H5FD_t *_f1, HDassert(finfo1 != NULL); HDassert(finfo2 != NULL); - if (finfo1->mKind != finfo2->mKind) HGOTO_DONE(-1); - if (finfo1->mName != finfo2->mName) HGOTO_DONE(-1); - if (finfo1->mLastMod != finfo2->mLastMod) HGOTO_DONE(-1); - if (finfo1->mSize != finfo2->mSize) HGOTO_DONE(-1); - if (finfo1->mReplication != finfo2->mReplication) HGOTO_DONE(-1); - if (finfo1->mBlockSize != finfo2->mBlockSize) HGOTO_DONE(-1); - if (strcmp(finfo1->mOwner, finfo2->mOwner)) HGOTO_DONE(-1); - if (strcmp(finfo1->mGroup, finfo2->mGroup)) HGOTO_DONE(-1); - if (finfo1->mPermissions != finfo2->mPermissions) HGOTO_DONE(-1); - if (finfo1->mLastAccess != finfo2->mLastAccess) HGOTO_DONE(-1); -#endif /* H5_HAVE_LIBHDFS */ + if (finfo1->mKind != finfo2->mKind) { HGOTO_DONE(-1); } + if (finfo1->mName != finfo2->mName) { HGOTO_DONE(-1); } + if (finfo1->mLastMod != finfo2->mLastMod) { HGOTO_DONE(-1); } + if (finfo1->mSize != finfo2->mSize) { HGOTO_DONE(-1); } + if (finfo1->mReplication != finfo2->mReplication) { HGOTO_DONE(-1); } + if (finfo1->mBlockSize != finfo2->mBlockSize) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mOwner, finfo2->mOwner)) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mGroup, finfo2->mGroup)) { HGOTO_DONE(-1); } + if (finfo1->mPermissions != finfo2->mPermissions) { HGOTO_DONE(-1); } + if (finfo1->mLastAccess != finfo2->mLastAccess) { HGOTO_DONE(-1); } done: FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_cmp() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static int +H5FD_hdfs_cmp( + const H5FD_t H5_ATTR_UNUSED *_f1, + const H5FD_t H5_ATTR_UNUSED *_f2) +{ + int ret_value = 0; + FUNC_ENTER_NOAPI_NOINIT_NOERR + + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_cmp() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * Function: H5FD_hdfs_query @@ -1580,7 +1599,7 @@ done: * Purpose: Set the flags that this VFL driver is capable of supporting. * (listed in H5FDpublic.h) * - * Note that since the HDFS VFD is read only, most flags + * Note that since the HDFS VFD is read only, most flags * are irrelevant. * * The term "set" is highly misleading... @@ -1595,7 +1614,7 @@ done: */ static herr_t H5FD_hdfs_query( - const H5FD_t H5_ATTR_UNUSED *_file, + const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) /* out variable */ { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1606,11 +1625,10 @@ H5FD_hdfs_query( if (flags) { *flags = 0; - *flags |= H5FD_FEAT_DATA_SIEVE; + *flags |= H5FD_FEAT_DATA_SIEVE; } FUNC_LEAVE_NOAPI(SUCCEED) - } /* H5FD_hdfs_query() */ @@ -1635,14 +1653,14 @@ H5FD_hdfs_query( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static haddr_t H5FD_hdfs_get_eoa( - const H5FD_t *_file, + const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { -#ifdef H5_HAVE_LIBHDFS const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1650,14 +1668,27 @@ H5FD_hdfs_get_eoa( HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); #endif -#ifdef H5_HAVE_LIBHDFS FUNC_LEAVE_NOAPI(file->eoa) -#else - FUNC_LEAVE_NOAPI(0) -#endif /* H5_HAVE_LIBHDFS */ +} /* end H5FD_hdfs_get_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eoa( + const H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(0) } /* end H5FD_hdfs_get_eoa() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1678,15 +1709,15 @@ H5FD_hdfs_get_eoa( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_set_eoa( - H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, + H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) { -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1694,16 +1725,30 @@ H5FD_hdfs_set_eoa( HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); #endif -#ifdef H5_HAVE_LIBHDFS file->eoa = addr; FUNC_LEAVE_NOAPI(SUCCEED) -#else - FUNC_LEAVE_NOAPI(FAIL) -#endif /* H5_HAVE_LIBHDFS */ +} /* H5FD_hdfs_set_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_set_eoa( + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t H5_ATTR_UNUSED addr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(FAIL) } /* H5FD_hdfs_set_eoa() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1715,7 +1760,7 @@ H5FD_hdfs_set_eoa( * * Return: * - * EOF: the first address past the end of the "file", either the + * EOF: the first address past the end of the "file", either the * filesystem file or the HDF5 file. * * Programmer: Jacob Smith @@ -1723,14 +1768,14 @@ H5FD_hdfs_set_eoa( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static haddr_t H5FD_hdfs_get_eof( - const H5FD_t *_file, + const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { -#ifdef H5_HAVE_LIBHDFS const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1738,17 +1783,30 @@ H5FD_hdfs_get_eof( HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); #endif -#ifdef H5_HAVE_LIBHDFS HDassert(file->hdfs_handle != NULL); HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); FUNC_LEAVE_NOAPI((size_t) file->hdfs_handle->fileinfo->mSize) -#else - FUNC_LEAVE_NOAPI((size_t)0) -#endif /* H5_HAVE_LIBHDFS */ +} /* end H5FD_hdfs_get_eof() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eof( + const H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); +#endif + + FUNC_LEAVE_NOAPI((size_t)0) } /* end H5FD_hdfs_get_eof() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1769,16 +1827,16 @@ H5FD_hdfs_get_eof( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_get_handle( - H5FD_t *_file, - hid_t H5_ATTR_UNUSED fapl, + H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, void **file_handle) { herr_t ret_value = SUCCEED; -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; -#endif FUNC_ENTER_NOAPI_NOINIT @@ -1786,29 +1844,48 @@ H5FD_hdfs_get_handle( HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); #endif /* HDFS_DEBUG */ -#ifdef H5_HAVE_LIBHDFS if (!file_handle) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") } *file_handle = file->hdfs_handle; -#else + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_get_handle() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_get_handle( + H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED fapl, + void H5_ATTR_UNUSED **file_handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); +#endif /* HDFS_DEBUG */ + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "Illegal get-handle of unsupported virtual file (hdfs)"); -#endif /* H5_HAVE_LIBHDFS */ done: FUNC_LEAVE_NOAPI(ret_value) - } /* end H5FD_hdfs_get_handle() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * * Function: H5FD_hdfs_read() * - * Purpose: + * Purpose: * * Reads SIZE bytes of data from FILE beginning at address ADDR * into buffer BUF according to data transfer properties in DXPL_ID. @@ -1828,20 +1905,20 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_read( - H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, + H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, /* start offset */ size_t size, /* length of read */ void *buf) /* out */ { herr_t ret_value = SUCCEED; -#if H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; size_t filesize = 0; -#endif /* H5_HAVE_LIBHDFS */ #if HDFS_STATS /* working variables for storing stats */ hdfs_statsbin *bin = NULL; @@ -1854,10 +1931,6 @@ H5FD_hdfs_read( HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); #endif /* HDFS_DEBUG */ -#ifndef H5_HAVE_LIBHDFS - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, - "Illegal get-handle of unsupported virtual file (hdfs)"); -#else HDassert(file != NULL); HDassert(file->hdfs_handle != NULL); HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); @@ -1894,37 +1967,64 @@ H5FD_hdfs_read( ? &file->raw[bin_i] : &file->meta[bin_i]; - /* Store collected stats in appropriate bin + /* Store collected stats in appropriate bin */ if (bin->count == 0) { bin->min = size; bin->max = size; - } else { - if (size < bin->min) bin->min = size; - if (size > bin->max) bin->max = size; + } + else { + if (size < bin->min) { bin->min = size; } + if (size > bin->max) { bin->max = size; } } bin->count++; bin->bytes += (unsigned long long)size; #endif /* HDFS_STATS */ -#endif /* H5_HAVE_LIBHDFS */ done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_read() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_read( + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, + void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); +#endif /* HDFS_DEBUG */ + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "Illegal get-handle of unsupported virtual file (hdfs)"); +done: + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_read() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * * Function: H5FD_hdfs_write() * - * Purpose: + * Purpose: * * Write bytes to file. * UNSUPPORTED IN READ-ONLY HDFS VFD. * - * Return: + * Return: * * FAIL (Not possible with Read-Only S3 file.) * @@ -1937,11 +2037,11 @@ done: */ static herr_t H5FD_hdfs_write( - H5FD_t H5_ATTR_UNUSED *_file, - H5FD_mem_t H5_ATTR_UNUSED type, + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, - haddr_t H5_ATTR_UNUSED addr, - size_t H5_ATTR_UNUSED size, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf) { herr_t ret_value = FAIL; @@ -1957,7 +2057,6 @@ H5FD_hdfs_write( done: FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_hdfs_write() */ @@ -1985,8 +2084,8 @@ done: */ static herr_t H5FD_hdfs_truncate( - H5FD_t H5_ATTR_UNUSED *_file, - hid_t H5_ATTR_UNUSED dxpl_id, + H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) { herr_t ret_value = SUCCEED; @@ -2002,7 +2101,6 @@ H5FD_hdfs_truncate( done: FUNC_LEAVE_NOAPI(ret_value) - } /* end H5FD_hdfs_truncate() */ @@ -2031,12 +2129,11 @@ done: */ static herr_t H5FD_hdfs_lock( - H5FD_t H5_ATTR_UNUSED *_file, + H5FD_t H5_ATTR_UNUSED *_file, hbool_t H5_ATTR_UNUSED rw) { FUNC_ENTER_NOAPI_NOINIT_NOERR FUNC_LEAVE_NOAPI(SUCCEED) - } /* end H5FD_hdfs_lock() */ @@ -2065,6 +2162,5 @@ H5FD_hdfs_unlock(H5FD_t H5_ATTR_UNUSED *_file) { FUNC_ENTER_NOAPI_NOINIT_NOERR FUNC_LEAVE_NOAPI(SUCCEED) - } /* end H5FD_hdfs_unlock() */ diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 86642aa..18f1f91 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -11,12 +11,12 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Read-Only S3 Virtual File Driver (VFD) + * Read-Only S3 Virtual File Driver (VFD) * * Programmer: Jacob Smith <jake.smith@hdfgroup.org> * 2017-10-13 * - * Purpose: + * Purpose: * * Provide read-only access to files hosted on Amazon's S3 service. * Relies on "s3comms" utility layer to implement the AWS REST API. @@ -137,9 +137,9 @@ typedef struct { * * Purpose: * - * H5FD_ros3_t is a structure used to store all information needed to + * H5FD_ros3_t is a structure used to store all information needed to * maintain R/O access to a single HDF5 file that has been stored as a - * S3 object. This structure is created when such a file is "opened" and + * S3 object. This structure is created when such a file is "opened" and * discarded when it is "closed". * * Presents an S3 object as a file to the HDF5 library. @@ -154,7 +154,7 @@ typedef struct { * * `fa` (H5FD_ros3_fapl_t) * - * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data + * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data * needed to "open" the HDF5 file. * * `eoa` (haddr_t) @@ -163,9 +163,9 @@ typedef struct { * equal the file size. * * `s3r_handle` (s3r_t *) - * + * * Instance of S3 Request handle associated with the target resource. - * Responsible for communicating with remote host and presenting file + * Responsible for communicating with remote host and presenting file * contents as indistinguishable from a file on the local filesystem. * * *** present only if ROS3_SATS is flagged to enable stats collection *** @@ -211,10 +211,13 @@ typedef struct H5FD_ros3_t { * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' * is too large to be represented by the second argument * of the file seek function. + * Only included if it may be used -- ROS3 VFD is enabled. * */ #define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#ifdef H5_HAVE_ROS3_VFD #define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#endif /* H5_HAVE_ROS3_VFD */ /* Prototypes */ static herr_t H5FD_ros3_term(void); @@ -229,13 +232,13 @@ static herr_t H5FD_ros3_query(const H5FD_t *_f1, unsigned long *flags); static haddr_t H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t type); static herr_t H5FD_ros3_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); static haddr_t H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, +static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); -static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, void *buf); -static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, const void *buf); -static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, +static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD_ros3_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD_ros3_unlock(H5FD_t *_file); @@ -276,8 +279,10 @@ static const H5FD_class_t H5FD_ros3_g = { H5FD_FLMAP_DICHOTOMY /* fl_map */ }; +#ifdef H5_HAVE_ROS3_VFD /* Declare a free list to manage the H5FD_ros3_t struct */ H5FL_DEFINE_STATIC(H5FD_ros3_t); +#endif /* H5_HAVE_ROS3_VFD */ /*------------------------------------------------------------------------- @@ -300,14 +305,14 @@ H5FD__init_package(void) FUNC_ENTER_STATIC if (H5FD_ros3_init() < 0) { - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize ros3 VFD") } done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ +} /* end H5FD__init_package() */ /*------------------------------------------------------------------------- @@ -330,7 +335,7 @@ done: hid_t H5FD_ros3_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + hid_t ret_value = H5I_INVALID_HID; FUNC_ENTER_NOAPI(FAIL) @@ -388,6 +393,7 @@ H5FD_ros3_term(void) H5FD_ROS3_g = 0; FUNC_LEAVE_NOAPI(SUCCEED) + } /* end H5FD_ros3_term() */ @@ -395,8 +401,8 @@ H5FD_ros3_term(void) * Function: H5Pset_fapl_ros3 * * Purpose: Modify the file access property list to use the H5FD_ROS3 - * driver defined in this source file. All driver specfic - * properties are passed in as a pointer to a suitably + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably * initialized instance of H5FD_ros3_fapl_t * * Return: SUCCEED/FAIL @@ -407,14 +413,12 @@ H5FD_ros3_term(void) *------------------------------------------------------------------------- */ herr_t -H5Pset_fapl_ros3(hid_t fapl_id, +H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa) { H5P_genplist_t *plist = NULL; /* Property list pointer */ herr_t ret_value = FAIL; - - FUNC_ENTER_API(FAIL) H5TRACE2("e", "i*x", fapl_id, fa); @@ -425,21 +429,21 @@ H5Pset_fapl_ros3(hid_t fapl_id, #endif plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); - if (plist == NULL) { + if (plist == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, \ "not a file access property list") } - if (FAIL == H5FD_ros3_validate_config(fa)) + if (FAIL == H5FD_ros3_validate_config(fa)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config") + } ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa); done: - FUNC_LEAVE_API(ret_value) -} /* H5Pset_fapl_ros3() */ +} /* end H5Pset_fapl_ros3() */ /*------------------------------------------------------------------------- @@ -449,12 +453,12 @@ done: * contains internally consistant data. Return SUCCEED if so, * and FAIL otherwise. * - * Note the difference between internally consistant and - * correct. As we will have to try to access the target + * Note the difference between internally consistant and + * correct. As we will have to try to access the target * object to determine whether the supplied data is correct, * we will settle for internal consistancy at this point * - * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally + * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally * consistant data, FAIL otherwise. * * Programmer: Jacob Smith @@ -494,7 +498,7 @@ H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_ros3_validate_config() */ +} /* end H5FD_ros3_validate_config() */ /*------------------------------------------------------------------------- @@ -515,12 +519,12 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_fapl_ros3(hid_t fapl_id, +H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out) { - const H5FD_ros3_fapl_t *fa; - H5P_genplist_t *plist = NULL; /* Property list pointer */ - herr_t ret_value = SUCCEED; /* Return value */ + const H5FD_ros3_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_API(FAIL) H5TRACE2("e", "i*x", fapl_id, fa_out); @@ -529,8 +533,9 @@ H5Pget_fapl_ros3(hid_t fapl_id, HDfprintf(stdout, "H5Pget_fapl_ros3() called.\n"); #endif - if (fa_out == NULL) + if (fa_out == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL") + } plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); if (plist == NULL) { @@ -552,7 +557,7 @@ H5Pget_fapl_ros3(hid_t fapl_id, done: FUNC_LEAVE_API(ret_value) -} /* H5Pget_fapl_ros3() */ +} /* end H5Pget_fapl_ros3() */ /*------------------------------------------------------------------------- @@ -583,7 +588,7 @@ H5FD_ros3_fapl_get(H5FD_t *_file) fa = (H5FD_ros3_fapl_t *)H5MM_calloc(sizeof(H5FD_ros3_fapl_t)); if (fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") } @@ -595,12 +600,13 @@ H5FD_ros3_fapl_get(H5FD_t *_file) done: if (ret_value == NULL) { - if (fa != NULL) - H5MM_xfree(fa); + if (fa != NULL) { + H5MM_xfree(fa); + } } FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_ros3_fapl_get() */ +} /* end H5FD_ros3_fapl_get() */ /*------------------------------------------------------------------------- @@ -624,13 +630,13 @@ H5FD_ros3_fapl_copy(const void *_old_fa) { const H5FD_ros3_fapl_t *old_fa = (const H5FD_ros3_fapl_t*)_old_fa; H5FD_ros3_fapl_t *new_fa = NULL; - void *ret_value = NULL; /* Return value */ + void *ret_value = NULL; FUNC_ENTER_NOAPI_NOINIT new_fa = (H5FD_ros3_fapl_t *)H5MM_malloc(sizeof(H5FD_ros3_fapl_t)); if (new_fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); } @@ -639,12 +645,13 @@ H5FD_ros3_fapl_copy(const void *_old_fa) done: if (ret_value == NULL) { - if (new_fa != NULL) + if (new_fa != NULL) { H5MM_xfree(new_fa); + } } FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_ros3_fapl_copy() */ +} /* end H5FD_ros3_fapl_copy() */ /*------------------------------------------------------------------------- @@ -668,13 +675,13 @@ H5FD_ros3_fapl_free(void *_fa) FUNC_ENTER_NOAPI_NOINIT_NOERR - HDassert(fa != NULL); /* sanity check */ + HDassert(fa != NULL); /* sanity check */ H5MM_xfree(fa); FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FD_ros3_fapl_free() */ +} /* end H5FD_ros3_fapl_free() */ #if ROS3_STATS @@ -707,7 +714,6 @@ ros3_reset_stats(H5FD_ros3_t *file) unsigned i = 0; herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT #if ROS3_DEBUG @@ -734,7 +740,8 @@ ros3_reset_stats(H5FD_ros3_t *file) done: FUNC_LEAVE_NOAPI(ret_value); -} /* ros3_reset_stats */ +} /* end ros3_reset_stats() */ + #endif /* ROS3_STATS */ @@ -747,10 +754,10 @@ done: * Create and/or opens a file as an HDF5 file. * * Any flag except H5F_ACC_RDONLY will cause an error. - * + * * Name (as received from `H5FD_open()`) must conform to web url: * NAME :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ] - * HTTP :: "http" [ "s" ] + * HTTP :: "http" [ "s" ] * DOMAIN :: e.g., "mybucket.host.org" * PORT :: ":" <number> (e.g., ":9000" ) * URI :: <string> (e.g., "path/to/resource.hd5" ) @@ -758,8 +765,8 @@ done: * * Return: * - * Success: A pointer to a new file data structure. - * The public fields will be initialized by the caller, which is + * Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is * always H5FD_open(). * * Failure: NULL @@ -771,32 +778,31 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + static H5FD_t * -H5FD_ros3_open(const char *url, - unsigned flags, - hid_t fapl_id, - haddr_t maxaddr) +H5FD_ros3_open( + const char *url, + unsigned flags, + hid_t fapl_id, + haddr_t maxaddr) { -#ifdef H5_HAVE_ROS3_VFD H5FD_ros3_t *file = NULL; struct tm *now = NULL; char iso8601now[ISO8601_SIZE]; unsigned char signing_key[SHA256_DIGEST_LENGTH]; s3r_t *handle = NULL; H5FD_ros3_fapl_t fa; -#endif H5FD_t *ret_value = NULL; FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_open() called.\n"); #endif - /* Sanity check on file offsets */ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); @@ -837,7 +843,7 @@ H5FD_ros3_open(const char *url, if (FAIL == H5FD_s3comms_signing_key(signing_key, (const char *)fa.secret_key, (const char *)fa.aws_region, - (const char *)iso8601now) ) + (const char *)iso8601now) ) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key") @@ -854,17 +860,17 @@ H5FD_ros3_open(const char *url, if (handle == NULL) { /* If we want to check CURL's say on the matter in a controlled - * fashion, this is the place to do it, but would need to make a + * fashion, this is the place to do it, but would need to make a * few minor changes to s3comms `s3r_t` and `s3r_read()`. - */ + */ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open"); } - /* create new file struct + /* create new file struct */ file = H5FL_CALLOC(H5FD_ros3_t); if (file == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") } @@ -873,7 +879,7 @@ H5FD_ros3_open(const char *url, #if ROS3_STATS if (FAIL == ros3_reset_stats(file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics") } #endif /* ROS3_STATS */ @@ -882,9 +888,9 @@ H5FD_ros3_open(const char *url, done: if (ret_value == NULL) { - if (handle != NULL) { + if (handle != NULL) { if (FAIL == H5FD_s3comms_s3r_close(handle)) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle") } } @@ -892,13 +898,28 @@ done: file = H5FL_FREE(H5FD_ros3_t, file); } curl_global_cleanup(); /* early cleanup because open failed */ - } /* if null return value (error) */ -#endif /* H5_HAVE_ROS3_VFD */ + } /* end if null return value (error) */ FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_ros3_open() */ + +#else /* H5_HAVE_ROS3_VFD not defined */ + +static H5FD_t * +H5FD_ros3_open( + const char H5_ATTR_UNUSED *url, + unsigned H5_ATTR_UNUSED flags, + hid_t H5_ATTR_UNUSED fapl_id, + haddr_t H5_ATTR_UNUSED maxaddr) +{ + H5FD_t *ret_value = NULL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_ros3_open() */ +#endif /* H5_HAVE_ROS3_VFD */ + #if ROS3_STATS /*---------------------------------------------------------------------------- @@ -906,7 +927,7 @@ done: * Function: ros3_fprint_stats() * * Purpose: - * + * * Tabulate and pretty-print statistics for this virtual file. * * Should be called upon file close. @@ -927,17 +948,17 @@ done: * - max (largest size read) * 2. tabulation of "bins", sepraring reads into exponentially-larger * ranges of size. - * - columns for number of reads, total bytes, and average size, with + * - columns for number of reads, total bytes, and average size, with * separate sub-colums for raw- and metadata reads. * - each row represents one bin, identified by the top of its range - * + * * Bin ranges can be modified with pound-defines at the top of this file. * * Bins without any reads in their bounds are not printed. * * An "overflow" bin is also present, to catch "big" reads. * - * Output for all bins (and range ceiling and average size report) + * Output for all bins (and range ceiling and average size report) * is divied by powers of 1024. By corollary, four digits before the decimal * is valid. * @@ -1050,7 +1071,7 @@ ros3_fprint_stats(FILE *stream, HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", bytes_raw + bytes_meta, bytes_meta, bytes_raw); - if (count_raw + count_meta == 0) + if (count_raw + count_meta == 0) goto done; /************************* @@ -1109,9 +1130,9 @@ ros3_fprint_stats(FILE *stream, * PRINT INDIVIDUAL BIN STATS * ******************************/ - HDfprintf(stream, + HDfprintf(stream, "BINS # of reads total bytes average size\n"); - HDfprintf(stream, + HDfprintf(stream, " up-to meta raw meta raw meta raw\n"); for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { @@ -1129,7 +1150,7 @@ ros3_fprint_stats(FILE *stream, m = &file->meta[i]; r = &file->raw[i]; - if (r->count == 0 && m->count == 0) + if (r->count == 0 && m->count == 0) continue; range_end = ros3_stats_boundaries[i]; @@ -1172,7 +1193,7 @@ ros3_fprint_stats(FILE *stream, re_dub /= 1024.0; HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, + HDfprintf(stream, " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, suffixes[suffix_i], /* bin ceiling */ m->count, /* metadata reads */ @@ -1187,7 +1208,7 @@ ros3_fprint_stats(FILE *stream, done: FUNC_LEAVE_NOAPI(ret_value); - + } /* ros3_fprint_stats */ #endif /* ROS3_STATS */ @@ -1201,7 +1222,7 @@ done: * Close an HDF5 file. * * Return: - * + * * SUCCEED/FAIL * * Programmer: Jacob Smith @@ -1211,57 +1232,63 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + static herr_t -H5FD_ros3_close(H5FD_t *_file) +H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file) { -#ifdef H5_HAVE_ROS3_VFD H5FD_ros3_t *file = (H5FD_ros3_t *)_file; herr_t ret_value = SUCCEED; -#else - herr_t ret_value = FAIL; -#endif - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_close() called.\n"); #endif - /* Sanity checks + /* Sanity checks */ HDassert(file != NULL); HDassert(file->s3r_handle != NULL); - /* Close the underlying request handle + /* Close the underlying request handle */ if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) { - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle") } #if ROS3_STATS /* TODO: mechanism to re-target stats printout */ - if (FAIL == ros3_fprint_stats(stdout, file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, + if (ros3_fprint_stats(stdout, file) == FAIL) { + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics") } #endif /* ROS3_STATS */ - /* Release the file info + /* Release the file info */ file = H5FL_FREE(H5FD_ros3_t, file); done: curl_global_cleanup(); /* cleanup to answer init on open */ -#endif /* H5_HAVE_ROS3_VFD */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_ros3_close() */ +#else /* H5_HAVE_ROS3_VFD not defined */ + +static herr_t +H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_ros3_close() */ + +#endif /* H5_HAVE_ROS3_VFD */ + /*------------------------------------------------------------------------- * @@ -1269,7 +1296,7 @@ done: * * Purpose: * - * Compares two files belonging to this driver using an arbitrary + * Compares two files belonging to this driver using an arbitrary * (but consistent) ordering: * * + url scheme @@ -1284,7 +1311,7 @@ done: * tl;dr -> check URL, check crentials * * Return: - * + * * - Equivalent: 0 * - Not Equivalent: -1 * @@ -1300,22 +1327,20 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + static int -H5FD_ros3_cmp(const H5FD_t *_f1, - const H5FD_t *_f2) +H5FD_ros3_cmp( + const H5FD_t *_f1, + const H5FD_t *_f2) { -#ifdef H5_HAVE_ROS3_VFD const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1; const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2; const parsed_url_t *purl1 = NULL; const parsed_url_t *purl2 = NULL; -#endif int ret_value = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR -#ifdef H5_HAVE_ROS3_VFD #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_cmp() called.\n"); @@ -1334,71 +1359,124 @@ H5FD_ros3_cmp(const H5FD_t *_f1, HDassert(purl2->host != NULL); /* URL: SCHEME */ - if (HDstrcmp(purl1->scheme, purl2->scheme)) HGOTO_DONE(-1); + if (HDstrcmp(purl1->scheme, purl2->scheme)) { + HGOTO_DONE(-1); + } /* URL: HOST */ - if (HDstrcmp(purl1->host, purl2->host)) HGOTO_DONE(-1); + if (HDstrcmp(purl1->host, purl2->host)) { + HGOTO_DONE(-1); + } /* URL: PORT */ if (purl1->port && purl2->port) { - if (HDstrcmp(purl1->port, purl2->port)) HGOTO_DONE(-1); - } else if (purl1->port) { + if (HDstrcmp(purl1->port, purl2->port)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->port) { HGOTO_DONE(-1); - } else if (purl2->port) { + } + else + if (purl2->port) { HGOTO_DONE(-1); } /* URL: PATH */ if (purl1->path && purl2->path) { - if (HDstrcmp(purl1->path, purl2->path)) HGOTO_DONE(-1); - } else if (purl1->path && !purl2->path) { + if (HDstrcmp(purl1->path, purl2->path)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->path && !purl2->path) { HGOTO_DONE(-1); - } else if (purl2->path && !purl1->path) { + } + else + if (purl2->path && !purl1->path) { HGOTO_DONE(-1); } /* URL: QUERY */ if (purl1->query && purl2->query) { - if (HDstrcmp(purl1->query, purl2->query)) HGOTO_DONE(-1); - } else if (purl1->query && !purl2->query) { + if (HDstrcmp(purl1->query, purl2->query)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->query && !purl2->query) { HGOTO_DONE(-1); - } else if (purl2->query && !purl1->query) { + } + else + if (purl2->query && !purl1->query) { HGOTO_DONE(-1); } /* FAPL: AWS_REGION */ if (f1->fa.aws_region[0] != '\0' && f1->fa.aws_region[0] != '\0') { - if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) HGOTO_DONE(-1); - } else if (f1->fa.aws_region[0] != '\0') { + if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.aws_region[0] != '\0') { HGOTO_DONE(-1); - } else if (f2->fa.aws_region[0] != '\0') { + } + else + if (f2->fa.aws_region[0] != '\0') { HGOTO_DONE(-1); } /* FAPL: SECRET_ID */ if (f1->fa.secret_id[0] != '\0' && f1->fa.secret_id[0] != '\0') { - if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) HGOTO_DONE(-1); - } else if (f1->fa.secret_id[0] != '\0') { + if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_id[0] != '\0') { HGOTO_DONE(-1); - } else if (f2->fa.secret_id[0] != '\0') { + } + else + if (f2->fa.secret_id[0] != '\0') { HGOTO_DONE(-1); } /* FAPL: SECRET_KEY */ if (f1->fa.secret_key[0] != '\0' && f1->fa.secret_key[0] != '\0') { - if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) HGOTO_DONE(-1); - } else if (f1->fa.secret_key[0] != '\0') { + if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_key[0] != '\0') { HGOTO_DONE(-1); - } else if (f2->fa.secret_key[0] != '\0') { + } + else + if (f2->fa.secret_key[0] != '\0') { HGOTO_DONE(-1); } -#endif /* H5_HAVE_ROS3_VFD */ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_ros3_cmp() */ +#else /* H5_HAVE_ROS3_VFD not defined */ + +static int +H5FD_ros3_cmp( + const H5FD_t H5_ATTR_UNUSED *_f1, + const H5FD_t H5_ATTR_UNUSED *_f2) +{ + int ret_value = 0; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_ros3_cmp() */ + +#endif /* H5_HAVE_ROS3_VFD */ + /*------------------------------------------------------------------------- * Function: H5FD_ros3_query @@ -1406,7 +1484,7 @@ done: * Purpose: Set the flags that this VFL driver is capable of supporting. * (listed in H5FDpublic.h) * - * Note that since the ROS3 VFD is read only, most flags + * Note that since the ROS3 VFD is read only, most flags * are irrelevant. * * The term "set" is highly misleading... @@ -1420,7 +1498,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, +H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out */) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1433,7 +1511,7 @@ H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, if (flags) { *flags = 0; /* OK to perform data sieving for faster raw data reads & writes */ - *flags |= H5FD_FEAT_DATA_SIEVE; + *flags |= H5FD_FEAT_DATA_SIEVE; } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) @@ -1463,7 +1541,7 @@ H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, *------------------------------------------------------------------------- */ static haddr_t -H5FD_ros3_get_eoa(const H5FD_t *_file, +H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; @@ -1499,8 +1577,8 @@ H5FD_ros3_get_eoa(const H5FD_t *_file, *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_set_eoa(H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, +H5FD_ros3_set_eoa(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) { H5FD_ros3_t *file = (H5FD_ros3_t *)_file; @@ -1528,7 +1606,7 @@ H5FD_ros3_set_eoa(H5FD_t *_file, * * Return: * - * EOF: the first address past the end of the "file", either the + * EOF: the first address past the end of the "file", either the * filesystem file or the HDF5 file. * * Programmer: Jacob Smith @@ -1537,7 +1615,7 @@ H5FD_ros3_set_eoa(H5FD_t *_file, *------------------------------------------------------------------------- */ static haddr_t -H5FD_ros3_get_eof(const H5FD_t *_file, +H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; @@ -1573,8 +1651,8 @@ H5FD_ros3_get_eof(const H5FD_t *_file, *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_get_handle(H5FD_t *_file, - hid_t H5_ATTR_UNUSED fapl, +H5FD_ros3_get_handle(H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, void **file_handle) { H5FD_ros3_t *file = (H5FD_ros3_t *)_file; @@ -1586,8 +1664,9 @@ H5FD_ros3_get_handle(H5FD_t *_file, HDfprintf(stdout, "H5FD_ros3_get_handle() called.\n"); #endif - if(!file_handle) + if (!file_handle) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") + } *file_handle = file->s3r_handle; @@ -1601,7 +1680,7 @@ done: * * Function: H5FD_ros3_read() * - * Purpose: + * Purpose * * Reads SIZE bytes of data from FILE beginning at address ADDR * into buffer BUF according to data transfer properties in DXPL_ID. @@ -1622,8 +1701,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_read(H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, +H5FD_ros3_read(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, /* start offset */ size_t size, /* length of read */ @@ -1631,13 +1710,13 @@ H5FD_ros3_read(H5FD_t *_file, { H5FD_ros3_t *file = (H5FD_ros3_t *)_file; size_t filesize = 0; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; #if ROS3_STATS /* working variables for storing stats */ ros3_statsbin *bin = NULL; unsigned bin_i = 0; #endif /* ROS3_STATS */ - + FUNC_ENTER_NOAPI_NOINIT @@ -1655,7 +1734,7 @@ H5FD_ros3_read(H5FD_t *_file, HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address") } - if (FAIL == H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) ) { + if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) { HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read") } @@ -1664,30 +1743,33 @@ H5FD_ros3_read(H5FD_t *_file, /* Find which "bin" this read fits in. Can be "overflow" bin. */ for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) { - if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) + if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) { break; + } } bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i]; - /* Store collected stats in appropriate bin + /* Store collected stats in appropriate bin */ if (bin->count == 0) { bin->min = size; bin->max = size; - } else { - if (size < bin->min) + } + else { + if (size < bin->min) { bin->min = size; - if (size > bin->max) + } + if (size > bin->max) { bin->max = size; + } } bin->count++; bin->bytes += (unsigned long long)size; #endif /* ROS3_STATS */ - done: FUNC_LEAVE_NOAPI(ret_value) @@ -1698,12 +1780,12 @@ done: * * Function: H5FD_ros3_write() * - * Purpose: + * Purpose: * * Write bytes to file. * UNSUPPORTED IN READ-ONLY ROS3 VFD. * - * Return: + * Return: * * FAIL (Not possible with Read-Only S3 file.) * @@ -1715,11 +1797,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file, - H5FD_mem_t H5_ATTR_UNUSED type, +H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, - haddr_t H5_ATTR_UNUSED addr, - size_t H5_ATTR_UNUSED size, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf) { herr_t ret_value = FAIL; @@ -1762,8 +1844,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, - hid_t H5_ATTR_UNUSED dxpl_id, +H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) { herr_t ret_value = SUCCEED; @@ -1807,7 +1889,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file, +H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file, hbool_t H5_ATTR_UNUSED rw) { FUNC_ENTER_NOAPI_NOINIT_NOERR diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 8c0515d..42b281f 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /***************************************************************************** - * Read-Only S3 Virtual File Driver (VFD) + * Read-Only S3 Virtual File Driver (VFD) * * Source for S3 Communications module * @@ -51,6 +51,8 @@ */ #define S3COMMS_DEBUG 0 +#ifdef H5_HAVE_ROS3_VFD + /* manipulate verbosity of CURL output * operates separately from S3COMMS_DEBUG * @@ -65,6 +67,8 @@ */ #define S3COMMS_MAX_RANGE_STRING_SIZE 128 +#endif /* H5_HAVE_ROS3_VFD */ + /******************/ /* Local Typedefs */ /******************/ @@ -111,7 +115,7 @@ herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle); /* Functions */ /*************/ - + /*---------------------------------------------------------------------------- * * Function: curlwritecallback() @@ -147,8 +151,9 @@ curlwritecallback(char *ptr, size_t product = (size * nmemb); size_t written = 0; - if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) + if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) { return written; + } if (size > 0) { HDmemcpy(&(sds->data[sds->size]), ptr, product); @@ -158,9 +163,9 @@ curlwritecallback(char *ptr, return written; -} /* curlwritecallback */ - +} /* end curlwritecallback() */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_hrb_node_set() @@ -223,9 +228,10 @@ curlwritecallback(char *ptr, *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_hrb_node_set(hrb_node_t **L, - const char *name, - const char *value) +H5FD_s3comms_hrb_node_set( + hrb_node_t **L, + const char *name, + const char *value) { size_t i = 0; char *valuecpy = NULL; @@ -238,8 +244,6 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, hbool_t is_looking = TRUE; herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -247,8 +251,9 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, HDprintf("NAME: %s\n", name); HDprintf("VALUE: %s\n", value); HDprintf("LIST:\n->"); - for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) + for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) { HDfprintf(stdout, "{%s}\n->", node_ptr->cat); + } HDprintf("(null)\n"); fflush(stdout); node_ptr = NULL; @@ -281,34 +286,37 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, * copies */ if (value != NULL) { - size_t valuelen = HDstrlen(value); - size_t catlen = namelen + valuelen + 2; /* HDstrlen(": ") -> +2 */ - int sprint_ret = 0; + int ret = 0; + size_t valuelen = HDstrlen(value); + size_t catlen = namelen + valuelen + 2; /* +2 from ": " */ + size_t catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */ + namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); if (namecpy == NULL) { HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for name copy.\n"); } - HDmemcpy(namecpy, name, namelen + 1); + HDmemcpy(namecpy, name, (namelen + 1)); valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1)); if (valuecpy == NULL) { HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for value copy.\n"); } - HDmemcpy(valuecpy, value, valuelen + 1); + HDmemcpy(valuecpy, value, (valuelen + 1)); - nvcat = (char *)H5MM_malloc(sizeof(char) * (catlen + 1)); + nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite); if (nvcat == NULL) { HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for concatenated string.\n"); } - sprint_ret = HDsnprintf(nvcat, (catlen + 1), "%s: %s", name, value); - if (sprint_ret <= 0) + ret = HDsnprintf(nvcat, catwrite, "%s: %s", name, value); + if (ret < 0 || (size_t)ret > catlen) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "error while concatenating `%s: %s", name, value); - HDassert( catlen == (size_t)sprint_ret ); + "cannot concatenate `%s: %s", name, value); + } + HDassert( catlen == HDstrlen(nvcat) ); /* create new_node, should we need it */ @@ -334,7 +342,8 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove node from empty list"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("CREATE NEW\n"); fflush(stdout); #endif @@ -406,7 +415,8 @@ HDprintf("FREEING POINTER\n"); fflush(stdout); HDprintf("FREEING WORKING LOWERNAME\n"); fflush(stdout); #endif H5MM_xfree(lowername); lowername = NULL; - } else { + } + else { #if S3COMMS_DEBUG HDprintf("MODIFY HEAD\n"); fflush(stdout); #endif @@ -428,14 +438,17 @@ HDprintf("MODIFY HEAD\n"); fflush(stdout); H5MM_xfree(new_node); new_node = NULL; } - } else if (strcmp(lowername, node_ptr->lowername) < 0) { + } + else + if (strcmp(lowername, node_ptr->lowername) < 0) { is_looking = FALSE; if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove a node 'before' head"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("PREPEND NEW HEAD\n"); fflush(stdout); #endif @@ -464,7 +477,8 @@ HDprintf("PREPEND NEW HEAD\n"); fflush(stdout); if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove absent node"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("APPEND A NODE\n"); fflush(stdout); #endif @@ -479,14 +493,17 @@ HDprintf("APPEND A NODE\n"); fflush(stdout); new_node->cat = nvcat; node_ptr->next = new_node; } - } else if (strcmp(lowername, node_ptr->next->lowername) < 0) { + } + else + if (strcmp(lowername, node_ptr->next->lowername) < 0) { is_looking = FALSE; if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove absent node"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("INSERT A NODE\n"); fflush(stdout); #endif @@ -502,7 +519,9 @@ HDprintf("INSERT A NODE\n"); fflush(stdout); new_node->next = node_ptr->next; node_ptr->next = new_node; } - } else if (strcmp(lowername, node_ptr->next->lowername) == 0) { + } + else + if (strcmp(lowername, node_ptr->next->lowername) == 0) { is_looking = FALSE; @@ -528,7 +547,8 @@ HDprintf("REMOVE A NODE\n"); fflush(stdout); H5MM_xfree(lowername); lowername = NULL; - } else { + } + else { #if S3COMMS_DEBUG HDprintf("MODIFY A NODE\n"); fflush(stdout); #endif @@ -552,23 +572,24 @@ HDprintf("MODIFY A NODE\n"); fflush(stdout); node_ptr->value = valuecpy; node_ptr->cat = nvcat; } - } else { + } + else { /**************** * KEEP LOOKING * ****************/ node_ptr = node_ptr->next; } - } + } /* end while is_looking */ done: if (ret_value == FAIL) { /* clean up */ - if (nvcat != NULL) H5MM_xfree(nvcat); - if (namecpy != NULL) H5MM_xfree(namecpy); - if (lowername != NULL) H5MM_xfree(lowername); - if (valuecpy != NULL) H5MM_xfree(valuecpy); + if (nvcat != NULL) { H5MM_xfree(nvcat); } + if (namecpy != NULL) { H5MM_xfree(namecpy); } + if (lowername != NULL) { H5MM_xfree(lowername); } + if (valuecpy != NULL) { H5MM_xfree(valuecpy); } if (new_node != NULL) { HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC ); new_node->magic += 1ul; @@ -577,11 +598,9 @@ done: } FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_hrb_node_set() */ -} /* H5FD_s3comms_hrb_node_set */ - - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_hrb_destroy() @@ -637,8 +656,6 @@ H5FD_s3comms_hrb_destroy(hrb_t **_buf) hrb_t *buf = NULL; herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -658,12 +675,11 @@ H5FD_s3comms_hrb_destroy(hrb_t **_buf) buf->magic += 1ul; H5MM_xfree(buf); *_buf = NULL; - } + } /* end if `_buf` has some value */ done: FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD_s3comms_hrb_destroy */ +} /* end H5FD_s3comms_hrb_destroy() */ /*---------------------------------------------------------------------------- @@ -728,8 +744,6 @@ H5FD_s3comms_hrb_init_request(const char *_verb, char *vrsn = NULL; size_t vrsnlen = 0; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -743,11 +757,13 @@ H5FD_s3comms_hrb_init_request(const char *_verb, /* populate valid NULLs with defaults */ - if (_verb == NULL) + if (_verb == NULL) { _verb = "GET"; + } - if (_http_version == NULL) + if (_http_version == NULL) { _http_version = "HTTP/1.1"; + } /* malloc space for and prepare structure */ @@ -765,28 +781,26 @@ H5FD_s3comms_hrb_init_request(const char *_verb, /* malloc and copy strings for the structure */ + reslen = HDstrlen(_resource); + if (_resource[0] == '/') { - reslen = HDstrlen(_resource) + 1; - res = (char *)H5MM_malloc(sizeof(char) * reslen); + res = (char *)H5MM_malloc(sizeof(char) * (reslen+1)); if (res == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string"); } - HDstrncpy(res, _resource, reslen); - } else { - int sprint_ret = 0; - reslen = HDstrlen(_resource) + 2; - res = (char *)H5MM_malloc(sizeof(char) * reslen); + HDmemcpy(res, _resource, (reslen+1)); + } + else { + res = (char *)H5MM_malloc(sizeof(char) * (reslen+2)); if (res == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, - "no space for resource string"); + "no space for resource string"); } - sprint_ret = HDsnprintf(res, reslen, "/%s", _resource); - if (sprint_ret <= 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, - "errro while appending resource string %s", _resource); - HDassert( (reslen - 1) == (size_t)sprint_ret ); - } /* start resource string with '/' */ + *res = '/'; + HDmemcpy((&res[1]), _resource, (reslen+1)); + HDassert( (reslen+1) == HDstrlen(res) ); + } /* end if (else resource string not starting with '/') */ verblen = HDstrlen(_verb) + 1; verb = (char *)H5MM_malloc(sizeof(char) * verblen); @@ -815,7 +829,6 @@ H5FD_s3comms_hrb_init_request(const char *_verb, ret_value = request; done: - /* if there is an error, clean up after ourselves */ if (ret_value == NULL) { @@ -826,8 +839,7 @@ done: } FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD_s3comms_hrb_init_request */ +} /* end H5FD_s3comms_hrb_init_request() */ @@ -835,7 +847,60 @@ done: * S3R FUNCTIONS ****************************************************************************/ +#ifndef H5_HAVE_ROS3_VFD +/* No-op, auto-fail, unused-variable implementations if ROS3 is not enabled */ + +herr_t +H5FD_s3comms_s3r_close(s3r_t H5_ATTR_UNUSED *handle) +{ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_s3r_close() */ + +size_t +H5FD_s3comms_s3r_get_filesize(s3r_t H5_ATTR_UNUSED *handle) +{ + size_t ret_value = 0; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_s3r_get_filesize() */ + +herr_t +H5FD_s3comms_s3r_getsize(s3r_t H5_ATTR_UNUSED *handle) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_s3r_getsize() */ + +s3r_t * +H5FD_s3comms_s3r_open( + const char H5_ATTR_UNUSED *url, + const char H5_ATTR_UNUSED *region, + const char H5_ATTR_UNUSED *id, + const unsigned char H5_ATTR_UNUSED *signing_key) +{ + s3r_t *ret_value = NULL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_s3r_open() */ +herr_t +H5FD_s3comms_s3r_read( + s3r_t H5_ATTR_UNUSED *handle, + haddr_t H5_ATTR_UNUSED offset, + size_t H5_ATTR_UNUSED len, + void H5_ATTR_UNUSED *dest) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_s3r_read() */ + +#else + + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_close() @@ -876,11 +941,8 @@ H5FD_s3comms_s3r_close(s3r_t *handle) { herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_s3r_close.\n"); #endif @@ -910,14 +972,11 @@ H5FD_s3comms_s3r_close(s3r_t *handle) H5MM_xfree(handle); -#endif /* H5_HAVE_ROS3_VFD */ - done: FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_s3comms_s3r_close */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_get_filesize() @@ -941,22 +1000,20 @@ done: *---------------------------------------------------------------------------- */ size_t -H5FD_s3comms_s3r_get_filesize(s3r_t *handle) { - +H5FD_s3comms_s3r_get_filesize(s3r_t *handle) +{ size_t ret_value = 0; FUNC_ENTER_NOAPI_NOINIT_NOERR -#ifdef H5_HAVE_ROS3_VFD - if (handle != NULL) + if (handle != NULL) { ret_value = handle->filesize; -#endif /* H5_HAVE_ROS3_VFD */ + } FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_s3comms_s3r_get_filesize */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_getsize() @@ -1143,7 +1200,7 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t\n"); } - if (content_length == 0 || + if (content_length == 0 || errno == ERANGE) /* errno set by strtoumax*/ { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -1187,7 +1244,7 @@ done: } /* H5FD_s3comms_s3r_getsize */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_open() @@ -1463,7 +1520,7 @@ done: } /* H5FD_s3comms_s3r_open */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_read() @@ -1621,7 +1678,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, if (len > 0) { rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ - S3COMMS_MAX_RANGE_STRING_SIZE ); + (S3COMMS_MAX_RANGE_STRING_SIZE+1) ); if (rangebytesstr == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "could not malloc range format string.\n"); @@ -1631,12 +1688,12 @@ H5FD_s3comms_s3r_read(s3r_t *handle, "bytes="H5_PRINTF_HADDR_FMT"-"H5_PRINTF_HADDR_FMT, offset, offset + len - 1); - if (ret == 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format HTTP Range value"); } else if (offset > 0) { rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ - S3COMMS_MAX_RANGE_STRING_SIZE); + (S3COMMS_MAX_RANGE_STRING_SIZE+1)); if (rangebytesstr == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "could not malloc range format string.\n"); @@ -1645,7 +1702,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, (S3COMMS_MAX_RANGE_STRING_SIZE), "bytes="H5_PRINTF_HADDR_FMT"-", offset); - if (ret == 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format HTTP Range value"); } @@ -1679,7 +1736,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, } else { /* authenticate request */ - char authorization[512]; + char authorization[512+1]; /* 512 := approximate max length... * 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request," * "SignedHeaders=,Signature=")> @@ -1689,10 +1746,10 @@ H5FD_s3comms_s3r_read(s3r_t *handle, * + 20 <max? len(region)> * + 128 <max? len(signed_headers)> */ - char buffer1[512]; /* -> Canonical Request -> Signature */ - char buffer2[256]; /* -> String To Sign -> Credential */ + char buffer1[512+1]; /* -> Canonical Request -> Signature */ + char buffer2[256+1]; /* -> String To Sign -> Credential */ char iso8601now[ISO8601_SIZE]; - char signed_headers[48]; + char signed_headers[48+1]; /* should be large enough for nominal listing: * "host;range;x-amz-content-sha256;x-amz-date" * + '\0', with "range;" possibly absent @@ -1821,8 +1878,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, /**** COMPUTE AUTHORIZATION ****/ if (FAIL == /* buffer1 -> canonical request */ - H5FD_s3comms_aws_canonical_request(buffer1, + H5FD_s3comms_aws_canonical_request( + buffer1, + 512, signed_headers, + 48, request) ) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -1858,23 +1918,25 @@ H5FD_s3comms_s3r_read(s3r_t *handle, HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string"); - ret = HDsnprintf(authorization, + ret = HDsnprintf( + authorization, 512, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2, signed_headers, buffer1); - if (ret == 0 || ret >= 512) + if (ret <= 0 || ret >= 512) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string"); + } /* append authorization header to http request buffer */ - if (FAIL == - H5FD_s3comms_hrb_node_set( - &headers, - "Authorization", - (const char *)authorization) ) + if (H5FD_s3comms_hrb_node_set( + &headers, + "Authorization", + (const char *)authorization) + == FAIL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set Authorization header") @@ -1914,17 +1976,18 @@ H5FD_s3comms_s3r_read(s3r_t *handle, /* finally, set http headers in curl handle */ - if (CURLE_OK != - curl_easy_setopt(curlh, - CURLOPT_HTTPHEADER, - curlheaders) ) + if (curl_easy_setopt( + curlh, + CURLOPT_HTTPHEADER, + curlheaders) + != CURLE_OK) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "error while setting CURL option " "(CURLOPT_HTTPHEADER). (placeholder flags)"); } - } /* if should authenticate (info provided) */ + } /* end if should authenticate (info provided) */ /******************* * PERFORM REQUEST * @@ -1983,14 +2046,16 @@ H5FD_s3comms_s3r_read(s3r_t *handle, HDfprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n"); if (sds == NULL) { HDfprintf(stderr, "sds is NULL!\n"); - } else { + } + else { HDfprintf(stderr, "sds: 0x%lx\n", (long long)sds); HDfprintf(stderr, "sds->size: %d\n", (int)sds->size); if (len > sds->size) { HDfprintf(stderr, "buffer overwrite\n"); } } - } else { + } + else { HDfprintf(stderr, "performed on entire file\n"); } #endif @@ -2028,29 +2093,30 @@ done: if (curlh != NULL) { /* clear any Range */ - if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) { HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_RANGE") + } /* clear headers */ - if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) { HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_HTTPHEADER") + } } #endif /* H5_HAVE_ROS3_VFD */ FUNC_LEAVE_NOAPI(ret_value); - } /* H5FD_s3comms_s3r_read */ - +#endif /* H5_HAVE_ROS3_VFD */ /**************************************************************************** * MISCELLANEOUS FUNCTIONS ****************************************************************************/ - + /*---------------------------------------------------------------------------- * * Function: gmnow() @@ -2079,14 +2145,14 @@ gmnow(void) struct tm *ret_value = NULL; /* Doctor assert, checks against error in time() */ - if ( (time_t)(-1) != time(now_ptr) ) + if ( (time_t)(-1) != time(now_ptr) ) { ret_value = gmtime(now_ptr); + } HDassert( ret_value != NULL ); return ret_value; - -} /* gmnow */ +} /* end gmnow() */ /*---------------------------------------------------------------------------- @@ -2131,16 +2197,23 @@ gmnow(void) *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, - char *signed_headers_dest, - hrb_t *http_request) +H5FD_s3comms_aws_canonical_request( + char *canonical_request_dest, + int _cr_size, + char *signed_headers_dest, + int _sh_size, + hrb_t *http_request) { hrb_node_t *node = NULL; const char *query_params = ""; /* unused at present */ herr_t ret_value = SUCCEED; - int ret = 0; /* return value of HDsnprintf */ - size_t len = 0; /* working string length variable */ - char tmpstr[256]; + int ret = 0; + size_t cr_size = (size_t)_cr_size; + size_t sh_size = (size_t)_sh_size; + size_t cr_len = 0; /* working length of canonical request str */ + size_t sh_len = 0; /* working length of signed headers str */ + char tmpstr[256+1]; + tmpstr[256] = 0; /* terminating NULL */ /* "query params" refers to the optional element in the URL, e.g. * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J @@ -2152,8 +2225,6 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, * VFD use-cases. */ - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -2178,55 +2249,71 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, /* HTTP verb, resource path, and query string lines */ - len = (HDstrlen(http_request->verb) + + cr_len = (HDstrlen(http_request->verb) + HDstrlen(http_request->resource) + HDstrlen(query_params) + - 3 ); - ret = HDsnprintf(canonical_request_dest, - len + 1, - "%s\n%s\n%s\n", - http_request->verb, - http_request->resource, - query_params); - if (ret == 0 || (size_t)ret > len) + (size_t)3); /* three newline chars */ + if (cr_len >= cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } + ret = HDsnprintf( /* TODO: compiler warning */ + canonical_request_dest, + (cr_size-1), + "%s\n%s\n%s\n", + http_request->verb, + http_request->resource, + query_params); + if (ret < 0 || (size_t)ret >= cr_size) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to compose canonical request first line"); + } /* write in canonical headers, building signed headers concurrently */ - node = http_request->first_header; /* assumed at first sorted */ + node = http_request->first_header; /* assumed sorted */ while (node != NULL) { - size_t join_len = 0; /* string len of joined header-value */ - - HDassert( node->magic == S3COMMS_HRB_NODE_MAGIC ); - - len = HDstrlen(node->lowername); - join_len = HDstrlen(node->value) + len + 2; /* +2 <- ":\n" */ - ret = HDsnprintf(tmpstr, - join_len + 1, /* +1 for null terminator */ - "%s:%s\n", - node->lowername, - node->value); - if (ret == 0 || (size_t)ret > join_len) + + HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC); + + ret = HDsnprintf( + tmpstr, + 256, + "%s:%s\n", + node->lowername, + node->value); + if (ret < 0 || ret >= 256) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s", node->lowername, node->value); + } + cr_len += HDstrlen(tmpstr); + if (cr_len + 1 > cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } strcat(canonical_request_dest, tmpstr); - len += 1; /* semicolon */ - ret = HDsnprintf(tmpstr, - len + 1, - "%s;", - node->lowername); - if (ret == 0 || (size_t)ret > len) + ret = HDsnprintf( + tmpstr, + 256, + "%s;", + node->lowername); + if (ret < 0 || ret >= 256) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "unable to append signed header %s", + "unable to append semicolon to lowername %s", node->lowername); + } + sh_len += HDstrlen(tmpstr); + if (sh_len + 1 > sh_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in signed headers"); + } strcat(signed_headers_dest, tmpstr); node = node->next; - } + } /* end while node is not NULL */ /* remove tailing ';' from signed headers sequence */ @@ -2243,10 +2330,9 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, done: FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_aws_canonical_request() */ -} /* H5FD_s3comms_aws_canonical_request */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_bytes_to_hex() @@ -2293,16 +2379,15 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_bytes_to_hex(char *dest, - const unsigned char *msg, - size_t msg_len, - hbool_t lowercase) +H5FD_s3comms_bytes_to_hex( + char *dest, + const unsigned char *msg, + size_t msg_len, + hbool_t lowercase) { size_t i = 0; herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -2319,24 +2404,23 @@ H5FD_s3comms_bytes_to_hex(char *dest, } for (i = 0; i < msg_len; i++) { - int chars_written = - HDsnprintf(&(dest[i * 2]), - 3, /* 'X', 'X', '\n' */ - (lowercase == TRUE) ? "%02x" - : "%02X", - msg[i]); - if (chars_written != 2) + int chars_written = HDsnprintf(&(dest[i * 2]), + 3, /* 'X', 'X', '\n' */ + (lowercase == TRUE) ? "%02x" : "%02X", + msg[i]); + if (chars_written != 2) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem while writing hex chars for %c", msg[i]); + } } done: FUNC_LEAVE_NOAPI(ret_value); -} /* H5FD_s3comms_bytes_to_hex */ - +} /* end H5FD_s3comms_bytes_to_hex() */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_free_purl() @@ -2369,19 +2453,19 @@ H5FD_s3comms_free_purl(parsed_url_t *purl) if (purl != NULL) { HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC ); - if (purl->scheme != NULL) H5MM_xfree(purl->scheme); - if (purl->host != NULL) H5MM_xfree(purl->host); - if (purl->port != NULL) H5MM_xfree(purl->port); - if (purl->path != NULL) H5MM_xfree(purl->path); - if (purl->query != NULL) H5MM_xfree(purl->query); + if (purl->scheme != NULL) { H5MM_xfree(purl->scheme); } + if (purl->host != NULL) { H5MM_xfree(purl->host); } + if (purl->port != NULL) { H5MM_xfree(purl->port); } + if (purl->path != NULL) { H5MM_xfree(purl->path); } + if (purl->query != NULL) { H5MM_xfree(purl->query); } purl->magic += 1ul; H5MM_xfree(purl); } FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FD_s3comms_free_purl */ - +} /* end H5FD_s3comms_free_purl() */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_HMAC_SHA256() @@ -2424,27 +2508,22 @@ H5FD_s3comms_free_purl(parsed_url_t *purl) * *---------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + herr_t -H5FD_s3comms_HMAC_SHA256(const unsigned char *key, - size_t key_len, - const char *msg, - size_t msg_len, - char *dest) +H5FD_s3comms_HMAC_SHA256( + const unsigned char *key, + size_t key_len, + const char *msg, + size_t msg_len, + char *dest) { -#ifdef H5_HAVE_ROS3_VFD unsigned char md[SHA256_DIGEST_LENGTH]; unsigned int md_len = SHA256_DIGEST_LENGTH; herr_t ret_value = SUCCEED; -#else - herr_t ret_value = FAIL; -#endif /* H5_HAVE_ROS3_VFD */ - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD - #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_HMAC_SHA256.\n"); #endif @@ -2462,24 +2541,39 @@ H5FD_s3comms_HMAC_SHA256(const unsigned char *key, md, &md_len); - if (FAIL == - H5FD_s3comms_bytes_to_hex(dest, - (const unsigned char *)md, - (size_t)md_len, - true)) + if (H5FD_s3comms_bytes_to_hex( + dest, + (const unsigned char *)md, + (size_t)md_len, + true) + == FAIL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not convert to hex string."); } -#endif /* H5_HAVE_ROS3_VFD */ - done: FUNC_LEAVE_NOAPI(ret_value); - } /* H5FD_s3comms_HMAC_SHA256 */ +#else /* H5_HAVE_ROS3_VFD not defined */ + +herr_t +H5FD_s3comms_HMAC_SHA256( + const unsigned char H5_ATTR_UNUSED *key, + size_t H5_ATTR_UNUSED key_len, + const char H5_ATTR_UNUSED *msg, + size_t H5_ATTR_UNUSED msg_len, + char H5_ATTR_UNUSED *dest) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_HMAC_SHA256() */ + +#endif /* H5_HAVE_ROS3_VFD */ + /*----------------------------------------------------------------------------- * * Function: H5FD__s3comms_load_aws_creds_from_file() @@ -2565,85 +2659,98 @@ H5FD__s3comms_load_aws_creds_from_file( #endif /* format target line for start of profile */ - if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) + if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format profile label") + } /* look for start of profile */ do { /* clear buffer */ - for (buffer_i=0; buffer_i < 128; buffer_i++) buffer[buffer_i] = 0; + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } line_buffer = fgets(line_buffer, 128, file); - if (line_buffer == NULL) /* reached end of file */ + if (line_buffer == NULL) { /* reached end of file */ goto done; + } } while (strncmp(line_buffer, profile_line, HDstrlen(profile_line))); /* extract credentials from lines */ do { - size_t setting_name_len = 0; - const char *setting_name = NULL; - char line_prefix[128]; - /* clear buffer */ - for (buffer_i=0; buffer_i < 128; buffer_i++) buffer[buffer_i] = 0; + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } /* collect a line from file */ line_buffer = fgets(line_buffer, 128, file); - if (line_buffer == NULL) + if (line_buffer == NULL) { goto done; /* end of file */ + } /* loop over names to see if line looks like assignment */ for (setting_i = 0; setting_i < setting_count; setting_i++) { + size_t setting_name_len = 0; + const char *setting_name = NULL; + char line_prefix[128]; + setting_name = setting_names[setting_i]; setting_name_len = HDstrlen(setting_name); - if (128 < HDsnprintf( - line_prefix, - setting_name_len+2, - "%s=", - setting_name)) + if (HDsnprintf(line_prefix, 128, "%s=", setting_name) < 0) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format line prefix") + } /* found a matching name? */ if (!HDstrncmp(line_buffer, line_prefix, setting_name_len + 1)) { found_setting = 1; /* skip NULL destination buffer */ - if (setting_pointers[setting_i] == NULL) + if (setting_pointers[setting_i] == NULL) { break; + } /* advance to end of name in string */ do { line_buffer++; } while (*line_buffer != 0 && *line_buffer != '='); - if (*line_buffer == 0 || *(line_buffer+1) == 0) + if (*line_buffer == 0 || *(line_buffer+1) == 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "incomplete assignment in file") + } line_buffer++; /* was pointing at '='; advance */ /* copy line buffer into out pointer */ - HDstrncpy(setting_pointers[setting_i], (const char *)line_buffer, - HDstrlen(line_buffer)); + if (HDstrncpy( + setting_pointers[setting_i], + (const char *)line_buffer, + HDstrlen(line_buffer)) + == NULL) + { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to copy line into pointer") + } /* "trim" tailing whitespace by replacing with null terminator*/ buffer_i = 0; - while (!isspace(setting_pointers[setting_i][buffer_i])) + while (!isspace(setting_pointers[setting_i][buffer_i])) { buffer_i++; + } setting_pointers[setting_i][buffer_i] = '\0'; break; /* have read setting; don't compare with others */ - } - } + } /* end if possible name match */ + } /* end for each setting name */ } while (found_setting); done: FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__s3comms_load_aws_creds_from_file() */ -} /* H5FD__s3comms_load_aws_creds_from_file */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_load_aws_profile() @@ -2688,6 +2795,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, FILE *credfile = NULL; char awspath[117]; char filepath[128]; + int ret = 0; FUNC_ENTER_NOAPI_NOINIT @@ -2696,47 +2804,61 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, #endif /* TODO: Windows and other path gotchas */ - if (117 < HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME"))) + ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")) ; + if (ret < 0 || (size_t)ret >= 117) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format home-aws path") - if (128 < HDsnprintf(filepath, 128, "%s%s", awspath, "credentials")) + } + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "credentials"); + if (ret < 0 || (size_t)ret >= 128) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format credentials path") + } credfile = fopen(filepath, "r"); if (credfile != NULL) { - if (FAIL == H5FD__s3comms_load_aws_creds_from_file( + if (H5FD__s3comms_load_aws_creds_from_file( credfile, profile_name, key_id_out, secret_access_key_out, - aws_region_out)) + aws_region_out) + == FAIL) + { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws credentials") - if (EOF == fclose(credfile)) + } + if (fclose(credfile) == EOF) { HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close credentials file") + } credfile = NULL; - } + } /* end if credential file opened */ - if (128 < HDsnprintf(filepath, 128, "%s%s", awspath, "config")) + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "config"); + if (ret < 0 || (size_t)ret >= 128) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format config path") + } credfile = fopen(filepath, "r"); if (credfile != NULL) { - if (FAIL == H5FD__s3comms_load_aws_creds_from_file( + if (H5FD__s3comms_load_aws_creds_from_file( credfile, profile_name, (*key_id_out == 0) ? key_id_out : NULL, (*secret_access_key_out == 0) ? secret_access_key_out : NULL, - (*aws_region_out == 0) ? aws_region_out : NULL)) + (*aws_region_out == 0) ? aws_region_out : NULL) + == FAIL) + { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws config") - if (EOF == fclose(credfile)) + } + if (fclose(credfile) == EOF) { HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close config file") + } credfile = NULL; - } + } /* end if credential file opened */ /* fail if not all three settings were loaded */ if (*key_id_out == 0 || @@ -2748,16 +2870,15 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, done: if (credfile != NULL) { - if (EOF == fclose(credfile)) + if (fclose(credfile) == EOF) { HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL, "problem error-closing aws configuration file") + } } - FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_load_aws_profile() */ -} /* H5FD_s3comms_load_aws_profile */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_nlowercase() @@ -2787,9 +2908,10 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_nlowercase(char *dest, - const char *s, - size_t len) +H5FD_s3comms_nlowercase( + char *dest, + const char *s, + size_t len) { herr_t ret_value = SUCCEED; @@ -2814,10 +2936,9 @@ H5FD_s3comms_nlowercase(char *dest, done: FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_nlowercase() */ -} /* H5FD_s3comms_nlowercase */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_parse_url() @@ -2852,8 +2973,9 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_parse_url(const char *str, - parsed_url_t **_purl) +H5FD_s3comms_parse_url( + const char *str, + parsed_url_t **_purl) { parsed_url_t *purl = NULL; /* pointer to new structure */ const char *tmpstr = NULL; /* working pointer in string */ @@ -2863,8 +2985,6 @@ H5FD_s3comms_parse_url(const char *str, unsigned int i = 0; herr_t ret_value = FAIL; - - FUNC_ENTER_NOAPI_NOINIT; #if S3COMMS_DEBUG @@ -2946,7 +3066,8 @@ H5FD_s3comms_parse_url(const char *str, tmpstr++; } tmpstr++; - } else { + } /* end if (IPv6) */ + else { while (0 != *tmpstr) { if (':' == *tmpstr || '/' == *tmpstr || @@ -2956,12 +3077,14 @@ H5FD_s3comms_parse_url(const char *str, } tmpstr++; } - } /* if IPv4 or IPv6 */ + } /* end else (IPv4) */ len = tmpstr - curstr; if (len == 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "HOST substring cannot be empty"); - } else if (len > urllen) { + } + else + if (len > urllen) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of HOST substring"); } @@ -2990,7 +3113,9 @@ H5FD_s3comms_parse_url(const char *str, if (len == 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "PORT element cannot be empty"); - } else if (len > urllen) { + } + else + if (len > urllen) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of PORT substring"); } @@ -3005,12 +3130,12 @@ H5FD_s3comms_parse_url(const char *str, */ purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); if (purl->port == NULL) { /* cannot malloc */ - HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, - "can't allocate space for PORT"); + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for PORT"); } (void)HDstrncpy(purl->port, curstr, (size_t)len); purl->port[len] = 0; - } /* if PORT element */ + } /* end if PORT element */ /************* * READ PATH * @@ -3040,7 +3165,7 @@ H5FD_s3comms_parse_url(const char *str, (void)HDstrncpy(purl->path, curstr, (size_t)len); purl->path[len] = 0; } - } /* if PATH element */ + } /* end if PATH element */ /************** * READ QUERY * @@ -3067,9 +3192,7 @@ H5FD_s3comms_parse_url(const char *str, } /* cannot malloc path pointer */ (void)HDstrncpy(purl->query, curstr, (size_t)len); purl->query[len] = 0; - } /* if QUERY exists */ - - + } /* end if QUERY exists */ *_purl = purl; ret_value = SUCCEED; @@ -3079,10 +3202,9 @@ done: H5FD_s3comms_free_purl(purl); } FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_parse_url() */ -} /* H5FD_s3comms_parse_url */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_percent_encode_char() @@ -3129,15 +3251,12 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_percent_encode_char(char *repr, - const unsigned char c, - size_t *repr_len) +H5FD_s3comms_percent_encode_char( + char *repr, + const unsigned char c, + size_t *repr_len) { - unsigned int acc = 0; unsigned int i = 0; - unsigned int k = 0; - unsigned int stack[4] = {0, 0, 0, 0}; - unsigned int stack_size = 0; int chars_written = 0; herr_t ret_value = SUCCEED; #if S3COMMS_DEBUG @@ -3145,8 +3264,6 @@ H5FD_s3comms_percent_encode_char(char *repr, unsigned char hex[3] = {0, 0, 0}; #endif - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -3172,13 +3289,19 @@ H5FD_s3comms_percent_encode_char(char *repr, #endif *repr_len = 3; chars_written = HDsnprintf(repr, 4, "%%%02X", c); - if (chars_written != 3) + if (chars_written < 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot write char %c", c); - } else { + } + } /* end if single-byte unicode char */ + else { /* multi-byte, multi-percent representation */ + unsigned int acc = 0; /* byte accumulator */ + unsigned int k = 0; /* uint character representation */ + unsigned int stack_size = 0; + unsigned char stack[4] = {0, 0, 0, 0}; #if S3COMMS_DEBUG HDfprintf(stdout, " MULTI-BYTE\n"); #endif @@ -3191,21 +3314,22 @@ H5FD_s3comms_percent_encode_char(char *repr, acc = k; acc >>= 6; /* cull least */ acc <<= 6; /* six bits */ - stack[stack_size++] = k - acc; /* max six-bit number */ + stack[stack_size++] = (unsigned char)(k - acc); k = acc >> 6; } while (k > 0); - /* now have "stack" of two to four six-bit numbers - * to be put into UTF-8 byte fields + /* `stack` now has two to four six-bit 'numbers' to be put into + * UTF-8 byte fields. */ #if S3COMMS_DEBUG HDfprintf(stdout, " STACK:\n {\n"); for (i = 0; i < stack_size; i++) { - H5FD_s3comms_bytes_to_hex((char *)hex, - (unsigned char *)(&stack[i]), - 1, - FALSE); + H5FD_s3comms_bytes_to_hex( + (char *)hex, + (&stack[i]), + 1, + FALSE); hex[2] = 0; HDfprintf(stdout, " %s,\n", hex); } @@ -3218,15 +3342,20 @@ H5FD_s3comms_percent_encode_char(char *repr, /* prepend 11[1[1]]0 to first byte */ /* 110xxxxx, 1110xxxx, or 11110xxx */ - acc = 0xC0; /* 2^7 + 2^6 -> 11000000 */ - acc += (stack_size > 2) ? 0x20 : 0; - acc += (stack_size > 3) ? 0x10 : 0; - stack_size -= 1; - chars_written = HDsnprintf(repr, 4, "%%%02X", acc + stack[stack_size]); - if (chars_written != 3) + acc = 0xC0; /* 0x11000000 */ + acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */ + acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */ + stack_size--; + chars_written = HDsnprintf( + repr, + 4, + "%%%02X", + (unsigned char)(acc + stack[stack_size])); + if (chars_written < 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot write char %c", c); + } *repr_len += 3; /************************ @@ -3235,25 +3364,27 @@ H5FD_s3comms_percent_encode_char(char *repr, /* 10xxxxxx */ for (i = 0; i < stack_size; i++) { - chars_written = HDsnprintf(&repr[i*3 + 3], - 4, - "%%%02X", - 128 + stack[stack_size - 1 - i]); - if (chars_written != 3) + chars_written = HDsnprintf( + &repr[i*3 + 3], + 4, + "%%%02X", + (unsigned char)(0x80 + stack[stack_size - 1 - i])); + if (chars_written < 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot write char %c", c); + } *repr_len += 3; - } - } + } /* end for each continuation byte */ + } /* end else (multi-byte) */ + *(repr + *repr_len) = '\0'; done: FUNC_LEAVE_NOAPI(ret_value); - } /* H5FD_s3comms_percent_encode_char */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_signing_key() @@ -3301,13 +3432,15 @@ done: * *---------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + herr_t -H5FD_s3comms_signing_key(unsigned char *md, - const char *secret, - const char *region, - const char *iso8601now) +H5FD_s3comms_signing_key( + unsigned char *md, + const char *secret, + const char *region, + const char *iso8601now) { -#ifdef H5_HAVE_ROS3_VFD char *AWS4_secret = NULL; size_t AWS4_secret_len = 0; unsigned char datekey[SHA256_DIGEST_LENGTH]; @@ -3315,16 +3448,9 @@ H5FD_s3comms_signing_key(unsigned char *md, unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH]; int ret = 0; /* return value of HDsnprintf */ herr_t ret_value = SUCCEED; -#else - herr_t ret_value = SUCCEED; -#endif /* H5_HAVE_ROS3_VFD */ - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD - #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_signing_key.\n"); #endif @@ -3356,10 +3482,11 @@ H5FD_s3comms_signing_key(unsigned char *md, /* prepend "AWS4" to start of the secret key */ ret = HDsnprintf(AWS4_secret, AWS4_secret_len,"%s%s", "AWS4", secret); - if ((size_t)ret != (AWS4_secret_len - 1)) + if ((size_t)ret != (AWS4_secret_len - 1)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem writing AWS4+secret `%s`", secret); + } /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest * we know digest length, so ignore via NULL @@ -3396,13 +3523,26 @@ H5FD_s3comms_signing_key(unsigned char *md, done: H5MM_xfree(AWS4_secret); -#endif /* H5_HAVE_ROS3_VFD */ - FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_signing_key() */ + +#else /* H5_HAVE_ROS3_VFD not defined */ -} /* H5FD_s3comms_signing_key */ +herr_t +H5FD_s3comms_signing_key( + unsigned char H5_ATTR_UNUSED *md, + const char H5_ATTR_UNUSED *secret, + const char H5_ATTR_UNUSED *region, + const char H5_ATTR_UNUSED *iso8601now) +{ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI_NOINIT_NOERR; + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_signing_key() */ +#endif /* H5_HAVE_ROS3_VFD */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_tostringtosign() @@ -3450,13 +3590,14 @@ done: * *---------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD herr_t -H5FD_s3comms_tostringtosign(char *dest, - const char *req, - const char *now, - const char *region) +H5FD_s3comms_tostringtosign( + char *dest, + const char *req, + const char *now, + const char *region) { -#ifdef H5_HAVE_ROS3_VFD unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1]; size_t d = 0; char day[9]; @@ -3465,15 +3606,9 @@ H5FD_s3comms_tostringtosign(char *dest, int ret = 0; /* HDsnprintf return value */ herr_t ret_value = SUCCEED; char tmp[128]; -#else - herr_t ret_value = FAIL; -#endif /* H5_HAVE_ROS3_VFD */ - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD - #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_tostringtosign.\n"); #endif @@ -3495,8 +3630,6 @@ H5FD_s3comms_tostringtosign(char *dest, "Region cannot be NULL.\n") } - - for (i = 0; i < 128; i++) { tmp[i] = '\0'; } @@ -3507,11 +3640,10 @@ H5FD_s3comms_tostringtosign(char *dest, HDstrncpy(day, now, 8); day[8] = '\0'; ret = HDsnprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region); - if (ret <= 0 || ret >= 127) + if (ret <= 0 || ret >= 127) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem adding day and region to string") - - + } HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); d = 17; @@ -3528,11 +3660,12 @@ H5FD_s3comms_tostringtosign(char *dest, HDstrlen(req), checksum); - if (FAIL == - H5FD_s3comms_bytes_to_hex(hexsum, - (const unsigned char *)checksum, - SHA256_DIGEST_LENGTH, - true)) + if (H5FD_s3comms_bytes_to_hex( + hexsum, + (const unsigned char *)checksum, + SHA256_DIGEST_LENGTH, + true) + == FAIL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not create hex string"); @@ -3544,14 +3677,27 @@ H5FD_s3comms_tostringtosign(char *dest, dest[d] = '\0'; -#endif /* H5_HAVE_ROS3_VFD */ - done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ros3_tostringtosign() */ + +#else /* H5_HAVE_ROS3_VFD not defined */ -} /* H5ros3_tostringtosign */ +herr_t +H5FD_s3comms_tostringtosign( + char H5_ATTR_UNUSED *dest, + const char H5_ATTR_UNUSED *req, + const char H5_ATTR_UNUSED *now, + const char H5_ATTR_UNUSED *region) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ros3_tostringtosign() */ +#endif /* H5_HAVE_ROS3_VFD */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_trim() @@ -3594,9 +3740,7 @@ H5FD_s3comms_trim(char *dest, size_t s_len, size_t *n_written) { - herr_t ret_value = SUCCEED; - - + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -3612,8 +3756,6 @@ H5FD_s3comms_trim(char *dest, s_len = 0; } - - if (s_len > 0) { /* Find first non-whitespace character from start; * reduce total length per character. @@ -3645,10 +3787,9 @@ H5FD_s3comms_trim(char *dest, done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_trim() */ -} /* H5FD_s3comms_trim */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_uriencode() @@ -3698,11 +3839,12 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_uriencode(char *dest, - const char *s, - size_t s_len, - hbool_t encode_slash, - size_t *n_written) +H5FD_s3comms_uriencode( + char *dest, + const char *s, + size_t s_len, + hbool_t encode_slash, + size_t *n_written) { char c = 0; size_t dest_off = 0; @@ -3712,20 +3854,20 @@ H5FD_s3comms_uriencode(char *dest, herr_t ret_value = SUCCEED; size_t s_off = 0; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG HDfprintf(stdout, "H5FD_s3comms_uriencode called.\n"); #endif - if (s == NULL) + if (s == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "source string cannot be NULL"); - if (dest == NULL) + } + if (dest == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "destination cannot be NULL"); + } /* Write characters to destination, converting to percent-encoded * "hex-utf-8" strings if necessary. @@ -3741,12 +3883,14 @@ H5FD_s3comms_uriencode(char *dest, (c == '/' && encode_slash == FALSE)) { dest[dest_off++] = c; - } else { + } + else { hex_off = 0; - if (FAIL == - H5FD_s3comms_percent_encode_char(hex_buffer, - (const unsigned char)c, - &hex_len)) + if (H5FD_s3comms_percent_encode_char( + hex_buffer, + (const unsigned char)c, + &hex_len) + == FAIL) { hex_buffer[0] = c; hex_buffer[1] = 0; @@ -3758,8 +3902,8 @@ H5FD_s3comms_uriencode(char *dest, for (hex_off = 0; hex_off < hex_len; hex_off++) { dest[dest_off++] = hex_buffer[hex_off]; } - } - } + } /* end else (not a regular character) */ + } /* end for each character */ if (dest_off < s_len) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -3769,7 +3913,6 @@ H5FD_s3comms_uriencode(char *dest, done: FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_s3comms_uriencode */ diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h index 93a5a81..16f6a8b 100644 --- a/src/H5FDs3comms.h +++ b/src/H5FDs3comms.h @@ -577,7 +577,9 @@ H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, H5_DLL struct tm * gmnow(void); herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, + int cr_size, char *signed_headers_dest, + int sh_size, hrb_t *http_request); H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest, diff --git a/test/hdfs.c b/test/hdfs.c index 57dbac9..ab39da6 100644 --- a/test/hdfs.c +++ b/test/hdfs.c @@ -11,9 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Read-Only HDFS Virtual File Driver (VFD) + * Read-Only HDFS Virtual File Driver (VFD) * - * Purpose: + * Purpose: * * Verify behavior for Read-Only HDFS VFD. * @@ -27,11 +27,10 @@ #include "H5FDhdfs.h" /* this file driver's utilities */ +#ifdef H5_HAVE_LIBHDFS #define HDFS_TEST_DEBUG 0 #define HDFS_TEST_MAX_BUF_SIZE 256 -#if 0 -#define S3_TEST_MAX_URL_SIZE 256 -#endif /* s3comms relic */ +#endif /* H5_HAVE_LIBHDFS */ /***************************************************************************** * @@ -39,7 +38,7 @@ * * Purpose: * - * 1) Upon test failure, goto-jump to single-location teardown in test + * 1) Upon test failure, goto-jump to single-location teardown in test * function. E.g., `error:` (consistency with HDF corpus) or * `failed:` (reflects purpose). * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. @@ -48,22 +47,22 @@ * requires reading of entire line to know whether this if/call is * part of the test setup, test operation, or a test unto itself. * 3) Provide testing macros with optional user-supplied failure message; - * if not supplied (NULL), generate comparison output in the spirit of + * if not supplied (NULL), generate comparison output in the spirit of * test-driven development. E.g., "expected 5 but was -3" * User messages clarify test's purpose in code, encouraging description * without relying on comments. * 4) Configurable expected-actual order in generated comparison strings. - * Some prefer `VERIFY(expected, actual)`, others + * Some prefer `VERIFY(expected, actual)`, others * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch * to satifsy both parties, assuming one paradigm per test file. * (One could #undef and redefine the flag through the file as desired, * but _why_.) * - * Provided as courtesy, per consideration for inclusion in the library + * Provided as courtesy, per consideration for inclusion in the library * proper. * * Macros: - * + * * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order * FAIL_IF() - check condition * FAIL_UNLESS() - check _not_ condition @@ -80,8 +79,8 @@ /*---------------------------------------------------------------------------- * * ifdef flag: JSVERIFY_EXP_ACT - * - * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) * *---------------------------------------------------------------------------- @@ -114,10 +113,10 @@ * * Macro: FAIL_IF() * - * Purpose: + * Purpose: * * Make tests more accessible and less cluttered than - * `if (thing == otherthing()) TEST_ERROR` + * `if (thing == otherthing()) TEST_ERROR` * paradigm. * * The following lines are roughly equivalent: @@ -148,7 +147,7 @@ if (condition) { \ * * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", * e.g., "a != b". - * + * * Opposite of FAIL_IF; fails if the given condition is _not_ true. * * `FAIL_IF( 5 != my_op() )` @@ -254,10 +253,10 @@ if (!(condition)) { \ * * Macro: JSVERIFY() * - * Purpose: + * Purpose: * * Verify that two long integers are equal. - * If unequal, print failure message + * If unequal, print failure message * (with `reason`, if not NULL; expected/actual if NULL) * and jump to `error` at end of function * @@ -277,10 +276,10 @@ if ((long)(actual) != (long)(expected)) { \ * * Macro: JSVERIFY_NOT() * - * Purpose: + * Purpose: * * Verify that two long integers are _not_ equal. - * If equal, print failure message + * If equal, print failure message * (with `reason`, if not NULL; expected/actual if NULL) * and jump to `error` at end of function * @@ -300,10 +299,10 @@ if ((long)(actual) == (long)(expected)) { \ * * Macro: JSVERIFY_STR() * - * Purpose: + * Purpose: * * Verify that two strings are equal. - * If unequal, print failure message + * If unequal, print failure message * (with `reason`, if not NULL; expected/actual if NULL) * and jump to `error` at end of function * @@ -319,8 +318,8 @@ if (strcmp((actual), (expected)) != 0) { \ } /* JSVERIFY_STR */ -#else -/* JSVERIFY_EXP_ACT not defined +#else +/* JSVERIFY_EXP_ACT not defined * * Repeats macros above, but with actual/expected parameters reversed. */ @@ -373,23 +372,30 @@ if (strcmp((actual), (expected)) != 0) { \ * OTHER MACROS AND DEFINITIONS * ********************************/ -/* copied from src/hdfs.c +/* copied from src/hdfs.c */ +#ifdef H5_HAVE_LIBHDFS #define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) -#define MAX_HDFS_NAMENODE_NAME 128 +#endif /* H5_HAVE_LIBHDFS */ + +#define HDFS_NAMENODE_NAME_MAX_SIZE 128 /******************************* * FILE-LOCAL GLOBAL VARIABLES * *******************************/ + +#ifdef H5_HAVE_LIBHDFS static const char filename_missing[] = "/tmp/missing.txt"; static const char filename_bard[] = "/tmp/t8.shakespeare.txt"; static const char filename_raven[] = "/tmp/Poe_Raven.txt"; static const char filename_example_h5[] = "/tmp/t.h5"; +#endif /* H5_HAVE_LIBHDFS */ + static H5FD_hdfs_fapl_t default_fa = { 1, /* fa version */ "localhost", /* namenode name */ 0, /* namenode port */ - "", /* user name */ + "", /* user name */ "", /* kerberos path */ 1024, /* buffer size */ }; @@ -403,7 +409,7 @@ static H5FD_hdfs_fapl_t default_fa = { * * Function: test_fapl_config_validation() * - * Purpose: + * Purpose: * * Test data consistency of fapl configuration. * Tests `H5FD_hdfs_validate_config` indirectly through `H5Pset_fapl_hdfs`. @@ -522,7 +528,7 @@ test_fapl_config_validation(void) for (i = 0; i < ncases; i++) { /*--------------- - * per-test setup + * per-test setup *--------------- */ case_ptr = &cases_arr[i]; @@ -535,7 +541,7 @@ test_fapl_config_validation(void) *----------------------------------- */ H5E_BEGIN_TRY { - /* `H5FD_hdfs_validate_config(...)` is static/private + /* `H5FD_hdfs_validate_config(...)` is static/private * to src/hdfs.c and cannot (and should not?) be tested directly? * Instead, validate config through public api. */ @@ -543,7 +549,7 @@ test_fapl_config_validation(void) } H5E_END_TRY; JSVERIFY( case_ptr->expected, success, case_ptr->msg ) - + /* Make sure we can get back what we put in. * Only valid if the fapl configuration does not result in error. */ @@ -566,7 +572,7 @@ test_fapl_config_validation(void) "streambuffer size mismatch" ) JSVERIFY_STR( config.namenode_name, fa_fetch.namenode_name, - NULL ) + NULL ) JSVERIFY_STR( config.user_name, fa_fetch.user_name, NULL ) @@ -599,7 +605,7 @@ error: } return 1; -} /* test_fapl_config_validation */ +} /* end test_fapl_config_validation() */ /*------------------------------------------------------------------------- @@ -608,7 +614,7 @@ error: * * Purpose: Tests the file handle interface for the HDFS driver. * - * For now, test only fapl & flags. Extend as the + * For now, test only fapl & flags. Extend as the * work on the VFD continues. * * Return: Success: 0 @@ -642,7 +648,7 @@ test_hdfs_fapl(void) TESTING("HDFS fapl "); - /* Set property list and file name for HDFS driver. + /* Set property list and file name for HDFS driver. */ fapl_id = H5Pcreate(H5P_FILE_ACCESS); FAIL_IF( fapl_id < 0 ) @@ -654,12 +660,12 @@ test_hdfs_fapl(void) /**************** * Check that the VFD feature flags are correct - * SPEC MAY CHANGE + * SPEC MAY CHANGE ******************/ FAIL_IF( H5FDdriver_query(driver_id, &driver_flags) < 0 ) - JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE), + JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE), "bit(s) in `driver_flags` must align with " "H5FD_FEAT_DATA_SIEVE" ) @@ -676,14 +682,14 @@ error: return 1; -} /* test_hdfs_fapl() */ +} /* end test_hdfs_fapl() */ /*--------------------------------------------------------------------------- * * Function: test_vfd_open() * - * Purpose: + * Purpose: * * Demonstrate/specify VFD-level "Open" failure cases * @@ -700,6 +706,16 @@ error: static int test_vfd_open(void) { + +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD-level open"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ @@ -709,7 +725,6 @@ test_vfd_open(void) #define FAPL_UNCONFIGURED -3 /* H5P_FILE_ACCESS */ #define FAPL_HDFS -4 -#ifdef H5_HAVE_LIBHDFS /************************* * test-local structures * *************************/ @@ -799,21 +814,14 @@ test_vfd_open(void) FALSE, }, }; -#endif /* H5_HAVE_LIBHDFS */ + unsigned i = 0; + unsigned failing_conditions_count = 10; H5FD_t *fd = NULL; hid_t fapl_hdfs = -1; hid_t fapl_unconfigured = -1; - unsigned i = 0; - unsigned failing_conditions_count = 10; TESTING("HDFS VFD-level open"); -#ifndef H5_HAVE_LIBHDFS - SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); - return 0; -#else fapl_unconfigured = H5Pcreate(H5P_FILE_ACCESS); FAIL_IF( fapl_unconfigured < 0 ) @@ -833,10 +841,13 @@ test_vfd_open(void) fd = NULL; - if (T.which_fapl == FAPL_UNCONFIGURED) + if (T.which_fapl == FAPL_UNCONFIGURED) { fapl_id = fapl_unconfigured; - else if (T.which_fapl == FAPL_HDFS) + } + else + if (T.which_fapl == FAPL_HDFS) { fapl_id = fapl_hdfs; + } #if HDFS_TEST_DEBUG HDfprintf(stderr, "testing: %s\n", T.message); @@ -852,10 +863,11 @@ test_vfd_open(void) HDfprintf(stderr, "\n!!!!! WARNING !!!!!\n" \ " Successful open of file on local system " \ "with non-HDFS VFD.\n"); - JSVERIFY(SUCCEED, H5FDclose(fd), + JSVERIFY(SUCCEED, H5FDclose(fd), "unable to close errant open"); fd = NULL; - } else { + } + else { JSVERIFY(1, 0, T.message); /* print message and fail */ } } @@ -867,14 +879,14 @@ test_vfd_open(void) HDfprintf(stderr, "nominal open\n"); #endif /* HDFS_TEST_DEBUG */ - /* finally, show that a file can be opened + /* finally, show that a file can be opened */ fd = H5FDopen( - filename_bard, - H5F_ACC_RDONLY, - fapl_hdfs, + filename_bard, + H5F_ACC_RDONLY, + fapl_hdfs, MAXADDR); - FAIL_IF( NULL == fd ) + FAIL_IF( NULL == fd ) /************ * TEARDOWN * @@ -895,20 +907,23 @@ test_vfd_open(void) PASSED(); return 0; -#endif /* H5_HAVE_LIBHDFS */ error: + /*********** * CLEANUP * ***********/ - if (fd) - (void)H5FDclose(fd); + if (fd) { + (void)H5FDclose(fd); + } H5E_BEGIN_TRY { - if (fapl_hdfs >= 0) + if (fapl_hdfs >= 0) { (void)H5Pclose(fapl_hdfs); - if (fapl_unconfigured >= 0) + } + if (fapl_unconfigured >= 0) { (void)H5Pclose(fapl_unconfigured); + } } H5E_END_TRY; return 1; @@ -917,14 +932,16 @@ error: #undef FAPL_UNCONFIGURED #undef FAPL_HDFS -} /* test_vfd_open */ +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_vfd_open() */ /*--------------------------------------------------------------------------- * * Function: test_eof_eoa() * - * Purpose: + * Purpose: * * Demonstrate behavior of get_eof, get_eoa, and set_eoa. * @@ -941,6 +958,15 @@ error: static int test_eof_eoa(void) { +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS eof/eoa gets and sets"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ @@ -958,16 +984,10 @@ test_eof_eoa(void) TESTING("HDFS eof/eoa gets and sets"); -#ifndef H5_HAVE_LIBHDFS - SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); - return 0; -#else /********* * SETUP * *********/ - + fapl_id = H5Pcreate(H5P_FILE_ACCESS); FAIL_IF( 0 > fapl_id ) FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) @@ -990,31 +1010,31 @@ test_eof_eoa(void) H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW), "mismatch between DEFAULT and RAW memory types" ) JSVERIFY( 0, - H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), "EoA should be unset by H5FDopen" ) /* set EoA below EoF */ - JSVERIFY( SUCCEED, - H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202), + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202), "unable to set EoA (lower)" ) - JSVERIFY( 5458199, - H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), "EoF changed" ) - JSVERIFY( 44442202, - H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + JSVERIFY( 44442202, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), "EoA unchanged" ) /* set EoA above EoF */ - JSVERIFY( SUCCEED, - H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012), + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012), "unable to set EoA (higher)" ) - JSVERIFY( 5458199, - H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), "EoF changed" ) - JSVERIFY( 6789012, - H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + JSVERIFY( 6789012, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), "EoA unchanged" ) /************ @@ -1029,9 +1049,9 @@ test_eof_eoa(void) PASSED(); return 0; -#endif /* H5_HAVE_LIBHDFS */ error: + /*********** * CLEANUP * ***********/ @@ -1039,7 +1059,7 @@ error: if (fd_shakespeare != NULL) { (void)H5FDclose(fd_shakespeare); } - if (fapl_id >= 0) { + if (fapl_id >= 0) { H5E_BEGIN_TRY { (void)H5Pclose(fapl_id); } H5E_END_TRY; @@ -1047,15 +1067,17 @@ error: return 1; -} /* test_eof_eoa */ +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_eof_eoa() */ /*----------------------------------------------------------------------------- * * Function: test_H5FDread_without_eoa_set_fails() - * + * * Purpose: - * + * * Demonstrate a not-obvious constraint by the library, preventing * file read before EoA is set * @@ -1067,26 +1089,27 @@ error: static int test_H5FDread_without_eoa_set_fails(void) { -#ifdef H5_HAVE_LIBHDFS +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD read-eoa temporal coupling library limitation"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + char buffer[HDFS_TEST_MAX_BUF_SIZE]; unsigned int i = 0; -#endif /* H5_HAVE_LIBHDFS */ H5FD_t *file_shakespeare = NULL; hid_t fapl_id = -1; TESTING("HDFS VFD read-eoa temporal coupling library limitation"); -#ifndef H5_HAVE_LIBHDFS - SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); - return 0; -#else /********* * SETUP * *********/ - /* create HDFS fapl + /* create HDFS fapl */ fapl_id = H5Pcreate(H5P_FILE_ACCESS); FAIL_IF( fapl_id < 0 ) @@ -1137,9 +1160,9 @@ test_H5FDread_without_eoa_set_fails(void) PASSED(); return 0; -#endif /* H5_HAVE_LIBHDFS */ error: + /*********** * CLEANUP * ***********/ @@ -1155,7 +1178,9 @@ error: return 1; -} /* test_H5FDread_without_eoa_set_fails */ +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_H5FDread_without_eoa_set_fails() */ @@ -1163,7 +1188,7 @@ error: * * Function: test_read() * - * Purpose: + * Purpose: * * Return: * @@ -1178,11 +1203,19 @@ error: static int test_read(void) { +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD read/range-gets"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ -#ifdef H5_HAVE_LIBHDFS /************************* * test-local structures * *************************/ @@ -1248,23 +1281,16 @@ test_read(void) herr_t open_return = FAIL; char buffer[HDFS_TEST_MAX_BUF_SIZE]; unsigned int i = 0; -#endif /* H5_HAVE_LIBHDFS */ H5FD_t *file_raven = NULL; hid_t fapl_id = -1; TESTING("HDFS VFD read/range-gets"); -#ifndef H5_HAVE_LIBHDFS - SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); - return 0; -#else /********* * SETUP * *********/ - /* create HDFS fapl + /* create HDFS fapl */ fapl_id = H5Pcreate(H5P_FILE_ACCESS); FAIL_IF( fapl_id < 0 ) @@ -1275,7 +1301,7 @@ test_read(void) buffer[i] = 0; } - /* open file + /* open file */ file_raven = H5FDopen( filename_raven, @@ -1301,7 +1327,7 @@ test_read(void) FAIL_IF( HDFS_TEST_MAX_BUF_SIZE < test.len ) /* buffer too small! */ - FAIL_IF( FAIL == + FAIL_IF( FAIL == H5FDset_eoa( file_raven, H5FD_MEM_DEFAULT, test.eoa_set) ) /* zero buffer contents */ @@ -1345,15 +1371,16 @@ test_read(void) PASSED(); return 0; -#endif /* H5_HAVE_LIBHDFS */ error: + /*********** * CLEANUP * ***********/ - if (file_raven != 0) + if (file_raven != 0) { (void)H5FDclose(file_raven); + } if (fapl_id >= 0) { H5E_BEGIN_TRY { (void)H5Pclose(fapl_id); @@ -1362,14 +1389,16 @@ error: return 1; -} /* test_read */ +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_read() */ /*--------------------------------------------------------------------------- * * Function: test_noops_and_autofails() * - * Purpose: + * Purpose: * * Demonstrate the unavailable and do-nothing routines unique to * Read-Only VFD. @@ -1391,6 +1420,15 @@ error: static int test_noops_and_autofails(void) { +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD always-fail and no-op routines"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ @@ -1413,13 +1451,7 @@ test_noops_and_autofails(void) * SETUP * *********/ -#ifndef H5_HAVE_LIBHDFS - SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); - return 0; -#else - /* create HDFS fapl + /* create HDFS fapl */ fapl_id = H5Pcreate(H5P_FILE_ACCESS); FAIL_IF( fapl_id < 0 ) @@ -1441,7 +1473,7 @@ test_noops_and_autofails(void) /* auto-fail calls to write and truncate */ H5E_BEGIN_TRY { - JSVERIFY( FAIL, + JSVERIFY( FAIL, H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data), "write must fail" ) } H5E_END_TRY; @@ -1485,9 +1517,9 @@ test_noops_and_autofails(void) PASSED(); return 0; -#endif /* H5_HAVE_LIBHDFS */ error: + /*********** * CLEANUP * ***********/ @@ -1503,14 +1535,16 @@ error: return 1; -} /* test_noops_and_autofails*/ +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_noops_and_autofails() */ /*--------------------------------------------------------------------------- * * Function: test_cmp() * - * Purpose: + * Purpose: * * Verify "file comparison" behavior. * @@ -1527,117 +1561,6 @@ error: static int test_cmp(void) { -#if 0 - /********************* - * test-local macros * - *********************/ - - /************************* - * test-local structures * - *************************/ - - /************************ - * test-local variables * - ************************/ - - H5FD_t *fd_raven = NULL; - H5FD_t *fd_shakes = NULL; - H5FD_t *fd_raven_2 = NULL; - hbool_t curl_ready = FALSE; - hid_t fapl_id = -1; - - - - TESTING("HDFS cmp (comparison)"); - - if (s3_test_credentials_loaded == 0) { - SKIPPED(); - puts(" s3 credentials are not loaded"); - fflush(stdout); - return 0; - } - - /********* - * SETUP * - *********/ - - FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) - curl_ready = TRUE; - - fapl_id = H5Pcreate(H5P_FILE_ACCESS); - FAIL_IF( 0 > fapl_id ) - JSVERIFY( SUCCEED, H5Pset_fapl_hdfs(fapl_id, &restricted_access_fa), NULL ) - - fd_raven = H5FDopen( - url_text_public, - H5F_ACC_RDONLY, - fapl_id, - HADDR_UNDEF); - FAIL_IF( NULL == fd_raven ) - - fd_shakes = H5FDopen( - url_text_restricted, - H5F_ACC_RDONLY, - fapl_id, - HADDR_UNDEF); - FAIL_IF( NULL == fd_shakes ) - - fd_raven_2 = H5FDopen( - url_text_public, - H5F_ACC_RDONLY, - fapl_id, - HADDR_UNDEF); - FAIL_IF( NULL == fd_raven_2 ) - - /********* - * TESTS * - *********/ - - JSVERIFY( 0, H5FDcmp(fd_raven, fd_raven_2), NULL ) - JSVERIFY( -1, H5FDcmp(fd_raven, fd_shakes), NULL ) - JSVERIFY( 1, H5FDcmp(fd_shakes, fd_raven_2), NULL ) - - /************ - * TEARDOWN * - ************/ - - FAIL_IF( FAIL == H5FDclose(fd_raven) ) - fd_raven = NULL; - FAIL_IF( FAIL == H5FDclose(fd_shakes) ) - fd_shakes = NULL; - FAIL_IF( FAIL == H5FDclose(fd_raven_2) ) - fd_raven_2 = NULL; - FAIL_IF( FAIL == H5Pclose(fapl_id) ) - fapl_id = -1; - - curl_global_cleanup(); - curl_ready = FALSE; - - PASSED(); - return 0; - -error: - /*********** - * CLEANUP * - ***********/ - - if (fd_raven != NULL) (void)H5FDclose(fd_raven); - if (fd_raven_2 != NULL) (void)H5FDclose(fd_raven_2); - if (fd_shakes != NULL) (void)H5FDclose(fd_shakes); - if (TRUE == curl_ready) curl_global_cleanup(); - if (fapl_id >= 0) { - H5E_BEGIN_TRY { - (void)H5Pclose(fapl_id); - } H5E_END_TRY; - } - -#else - /* - JSFAILED_AT() - HDprintf("TODO\n"); - return 1; - */ - TESTING("HDFS cmp (comparison)"); SKIPPED(); HDfprintf( @@ -1645,16 +1568,15 @@ error: " TODO: Distinct valid fapls to open the same file.\n"); return 0; -#endif /* s3comms relic */ -} /* test_cmp */ +} /* end test_cmp() */ /*--------------------------------------------------------------------------- * * Function: test_H5F_integration() * - * Purpose: + * Purpose: * * Demonstrate H5F (File interface) behavior with files on HDFS. * @@ -1675,6 +1597,15 @@ error: static int test_H5F_integration(void) { +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS file access through HD5F library (H5F API)"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ @@ -1696,12 +1627,6 @@ test_H5F_integration(void) * SETUP * *********/ -#ifndef H5_HAVE_LIBHDFS - SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); - return 0; -#else fapl_id = H5Pcreate(H5P_FILE_ACCESS); FAIL_IF( 0 > fapl_id ) FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) @@ -1749,7 +1674,6 @@ test_H5F_integration(void) PASSED(); return 0; -#endif /* H5_HAVE_LIBHDFS */ error: /*********** @@ -1771,6 +1695,8 @@ error: return 1; +#endif /* H5_HAVE_LIBHDFS */ + } /* test_H5F_integration */ @@ -1796,15 +1722,19 @@ main(void) /****************** * commence tests * ******************/ - - static char hdfs_namenode_name[MAX_HDFS_NAMENODE_NAME] = ""; + + static char hdfs_namenode_name[HDFS_NAMENODE_NAME_MAX_SIZE] = ""; const char *hdfs_namenode_name_env = NULL; hdfs_namenode_name_env = HDgetenv("HDFS_TEST_NAMENODE_NAME"); if (hdfs_namenode_name_env == NULL || hdfs_namenode_name_env[0] == '\0') { - HDstrncpy(hdfs_namenode_name, "localhost", 9); - } else { - HDstrncpy(default_fa.namenode_name, hdfs_namenode_name_env, MAX_HDFS_NAMENODE_NAME); + HDstrncpy(hdfs_namenode_name, "localhost", HDFS_NAMENODE_NAME_MAX_SIZE); + } + else { + HDstrncpy( /* TODO: error-check? */ + default_fa.namenode_name, + hdfs_namenode_name_env, + HDFS_NAMENODE_NAME_MAX_SIZE); } h5_reset(); @@ -1823,14 +1753,15 @@ main(void) if (nerrors > 0) { HDprintf("***** %d hdfs TEST%s FAILED! *****\n", - nerrors, + nerrors, nerrors > 1 ? "S" : ""); nerrors = 1; - } else { + } + else { HDprintf("All hdfs tests passed.\n"); } return nerrors; /* 0 if no errors, 1 if any errors */ -} /* main() */ +} /* end main() */ diff --git a/test/ros3.c b/test/ros3.c index 60d3932..6d58af0 100644 --- a/test/ros3.c +++ b/test/ros3.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Read-Only S3 Virtual File Driver (VFD) + * Read-Only S3 Virtual File Driver (VFD) * * Purpose: * @@ -32,6 +32,9 @@ +/* only include the testing macros if needed */ +#ifdef H5_HAVE_ROS3_VFD + /***************************************************************************** * * FILE-LOCAL TESTING MACROS @@ -368,15 +371,16 @@ if (strcmp((actual), (expected)) != 0) { \ #endif /* ifdef/else JSVERIFY_EXP_ACT */ +#endif /* H5_HAVE_ROS3_VFD */ + /******************************** * OTHER MACROS AND DEFINITIONS * ********************************/ -/* copied from src/ros3.c - */ +#ifdef H5_HAVE_ROS3_VFD + #define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) -#ifdef H5_HAVE_ROS3_VFD #define S3_TEST_PROFILE_NAME "ros3_vfd_test" #define S3_TEST_MAX_URL_SIZE 256 @@ -440,6 +444,15 @@ H5FD_ros3_fapl_t anonymous_fa = { static int test_fapl_config_validation(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 fapl configuration validation"); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else /* H5_HAVE_ROS3_VFD defined */ + /********************* * test-local macros * *********************/ @@ -458,7 +471,6 @@ test_fapl_config_validation(void) * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD hid_t fapl_id = -1; /* file access property list ID */ H5FD_ros3_fapl_t config; H5FD_ros3_fapl_t fa_fetch; @@ -544,21 +556,12 @@ test_fapl_config_validation(void) }, }; -#endif /* H5_HAVE_ROS3_VFD */ - TESTING("ROS3 fapl configuration validation"); /********* * TESTS * *********/ -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else - if (FALSE == s3_test_bucket_defined) { SKIPPED(); puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); @@ -655,7 +658,7 @@ error: * Purpose: Tests the file handle interface for the ROS3 driver * * As the ROS3 driver is 1) read only, 2) requires access - * to an S3 server (minio for now), this test is quite + * to an S3 server, this test is quite * different from the other tests. * * For now, test only fapl & flags. Extend as the @@ -675,11 +678,19 @@ error: static int test_ros3_fapl(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 fapl "); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else /* H5_HAVE_ROS3 defined */ + /************************ * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD hid_t fapl_id = -1; /* file access property list ID */ hid_t driver_id = -1; /* ID for this VFD */ unsigned long driver_flags = 0; /* VFD feature flags */ @@ -690,16 +701,9 @@ test_ros3_fapl(void) "", /* secret_id */ "plugh", /* secret_key */ }; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("ROS3 fapl "); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else /* Set property list and file name for ROS3 driver. */ fapl_id = H5Pcreate(H5P_FILE_ACCESS); @@ -759,11 +763,19 @@ error: static int test_vfd_open(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 VFD-level open"); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else /* H5_HAVE_ROS3_VFD defined */ + /********************* * test-local macros * *********************/ -#ifdef H5_HAVE_ROS3_VFD #define FAPL_H5P_DEFAULT -2 #define FAPL_FILE_ACCESS -3 @@ -853,17 +865,9 @@ test_vfd_open(void) hid_t fapl_file_access = -1; unsigned i = 0; unsigned tests_count = 10; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("ROS3 VFD-level open"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else - if (FALSE == s3_test_bucket_defined) { SKIPPED(); puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); @@ -989,6 +993,15 @@ error: static int test_eof_eoa(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 eof/eoa gets and sets"); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else /* H5_HAVE_ROS3_VFD defined */ + /********************* * test-local macros * *********************/ @@ -1001,20 +1014,12 @@ test_eof_eoa(void) * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD H5FD_t *fd_shakespeare = NULL; hbool_t curl_ready = FALSE; hid_t fapl_id = -1; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("ROS3 eof/eoa gets and sets"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else if (s3_test_credentials_loaded == 0) { SKIPPED(); puts(" s3 credentials are not loaded"); @@ -1136,21 +1141,22 @@ error: static int test_H5FDread_without_eoa_set_fails(void) { -#ifdef H5_HAVE_ROS3_VFD +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 VFD read-eoa temporal coupling library limitation "); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else /* H5_HAVE_ROS3_VFD defined */ + char buffer[256]; unsigned int i = 0; H5FD_t *file_shakespeare = NULL; hid_t fapl_id = -1; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("ROS3 VFD read-eoa temporal coupling library limitation "); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else if (s3_test_credentials_loaded == 0) { SKIPPED(); puts(" s3 credentials are not loaded"); @@ -1256,6 +1262,15 @@ error: static int test_read(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 VFD read/range-gets"); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else /* H5_HAVE_ROS3_VFD defined */ + /********************* * test-local macros * *********************/ @@ -1275,7 +1290,6 @@ test_read(void) /************************ * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD struct testcase cases[] = { { "successful range-get", 6464, @@ -1328,16 +1342,9 @@ test_read(void) unsigned int i = 0; H5FD_t *file_raven = NULL; hid_t fapl_id = -1; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("ROS3 VFD read/range-gets"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else if (s3_test_credentials_loaded == 0) { SKIPPED(); puts(" s3 credentials are not loaded"); @@ -1471,6 +1478,15 @@ error: static int test_noops_and_autofails(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 VFD always-fail and no-op routines"); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ @@ -1483,21 +1499,13 @@ test_noops_and_autofails(void) * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD hbool_t curl_ready = FALSE; hid_t fapl_id = -1; H5FD_t *file = NULL; const char data[36] = "The Force shall be with you, always"; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("ROS3 VFD always-fail and no-op routines"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else if (FALSE == s3_test_bucket_defined) { SKIPPED(); @@ -1624,6 +1632,15 @@ error: static int test_cmp(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("ROS3 cmp (comparison)"); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ @@ -1636,22 +1653,14 @@ test_cmp(void) * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD H5FD_t *fd_raven = NULL; H5FD_t *fd_shakes = NULL; H5FD_t *fd_raven_2 = NULL; hbool_t curl_ready = FALSE; hid_t fapl_id = -1; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("ROS3 cmp (comparison)"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else if (s3_test_credentials_loaded == 0) { SKIPPED(); puts(" s3 credentials are not loaded"); @@ -1767,6 +1776,15 @@ error: static int test_H5F_integration(void) { +#ifndef H5_HAVE_ROS3_VFD + TESTING("S3 file access through HD5F library (H5F API)"); + SKIPPED(); + puts(" ROS3 VFD not enabled"); + fflush(stdout); + return 0; + +#else + /********************* * test-local macros * *********************/ @@ -1779,19 +1797,11 @@ test_H5F_integration(void) * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD hid_t file = -1; hid_t fapl_id = -1; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("S3 file access through HD5F library (H5F API)"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD not enabled"); - fflush(stdout); - return 0; -#else if (s3_test_credentials_loaded == 0) { SKIPPED(); puts(" s3 credentials are not loaded"); diff --git a/test/s3comms.c b/test/s3comms.c index a797e04..d636550 100644 --- a/test/s3comms.c +++ b/test/s3comms.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Read-Only S3 Virtual File Driver (VFD) + * Read-Only S3 Virtual File Driver (VFD) * * Purpose: Unit tests for the S3 Communications (s3comms) module. * @@ -386,7 +386,7 @@ static hbool_t s3_test_bucket_defined = FALSE; #endif /* H5_HAVE_ROS3_VFD */ - + /*--------------------------------------------------------------------------- * * Function: test_macro_format_credential() @@ -408,7 +408,6 @@ test_macro_format_credential(void) * test-local variables * ************************/ -#ifdef H5_HAVE_ROS3_VFD char dest[256]; const char access[] = "AKIAIOSFODNN7EXAMPLE"; const char date[] = "20130524"; @@ -416,16 +415,9 @@ test_macro_format_credential(void) const char service[] = "s3"; const char expected[] = "AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("test_macro_format_credential"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD is not enabled"); - fflush(stdout); - return 0; -#else FAIL_IF( S3COMMS_MAX_CREDENTIAL_SIZE < S3COMMS_FORMAT_CREDENTIAL(dest, access, date, region, service) ) @@ -433,14 +425,13 @@ test_macro_format_credential(void) PASSED(); return 0; -#endif /* H5_HAVE_ROS3_VFD */ error: return -1; -} /* test_macro_format_credential */ - +} /* end test_macro_format_credential() */ + /*--------------------------------------------------------------------------- * * Function: test_aws_canonical_request() @@ -465,7 +456,6 @@ test_aws_canonical_request(void) * test-local structures * *************************/ -#ifdef H5_HAVE_ROS3_VFD struct header { const char *name; const char *value; @@ -520,16 +510,9 @@ test_aws_canonical_request(void) hrb_node_t *node = NULL; /* http headers list pointer */ unsigned int n_cases = 3; char sh_dest[64]; /* signed headers */ -#endif /* H5_HAVE_ROS3_VFD */ TESTING("test_aws_canonical_request"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD is not enabled"); - fflush(stdout); - return 0; -#else for (i = 0; i < n_cases; i++) { /* pre-test bookkeeping */ @@ -559,16 +542,22 @@ test_aws_canonical_request(void) /* test */ JSVERIFY( SUCCEED, - H5FD_s3comms_aws_canonical_request(cr_dest, sh_dest, hrb), + H5FD_s3comms_aws_canonical_request( + cr_dest, + 512, + sh_dest, + 64, + hrb), " unable to compose canonical request" ) JSVERIFY_STR( C->exp_headers, sh_dest, NULL ) JSVERIFY_STR( C->exp_request, cr_dest, NULL ) /* tear-down */ - while (node != NULL) + while (node != NULL) { FAIL_IF( FAIL == H5FD_s3comms_hrb_node_set(&node, node->name, NULL)); + } HDassert(NULL == node); FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb)); HDassert(NULL == hrb); @@ -581,14 +570,29 @@ test_aws_canonical_request(void) /* malformed hrb and/or node-list */ - JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(cr_dest, sh_dest, NULL), + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request( + cr_dest, + 20, + sh_dest, + 20, + NULL), "http request object cannot be null" ) hrb = H5FD_s3comms_hrb_init_request("GET", "/", "HTTP/1.1"); - JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(NULL, sh_dest, hrb), + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request( + NULL, + 20, + sh_dest, + 20, + hrb), "canonical request destination cannot be NULL" ) - JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(cr_dest, NULL, hrb), + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request( + cr_dest, + 20, + NULL, + 20, + hrb), "signed headers destination cannot be null" ) FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb) ) @@ -604,15 +608,15 @@ error: (void)H5FD_s3comms_hrb_node_set(&node, node->name, NULL); HDassert( node == NULL ); } - if (hrb != NULL) + if (hrb != NULL) { (void)H5FD_s3comms_hrb_destroy(&hrb); + } return -1; -#endif /* H5_HAVE_ROS3_VFD */ - -} /* test_aws_canonical_request */ +} /* end test_aws_canonical_request() */ + /*--------------------------------------------------------------------------- * * Function: test_bytes_to_hex @@ -706,9 +710,9 @@ test_bytes_to_hex(void) error: return -1; -} /* test_bytes_to_hex */ - +} /* end test_bytes_to_hex() */ + /*--------------------------------------------------------------------------- * * Function: test_hrb_init_request() @@ -725,7 +729,6 @@ error: static herr_t test_hrb_init_request(void) { -#ifdef H5_HAVE_ROS3_VFD /********************* * test-local macros * *********************/ @@ -769,7 +772,7 @@ test_hrb_init_request(void) "HTTP/1.1", FALSE, }, - { "slash prepented to resource path, if necessary", + { "slash prepended to resource path, if necessary", NULL, "MYPATH/MYFILE.tiff", "/MYPATH/MYFILE.tiff", @@ -788,29 +791,25 @@ test_hrb_init_request(void) unsigned int i = 0; unsigned int ncases = 5; hrb_t *req = NULL; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("hrb_init_request"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD is not enabled"); - fflush(stdout); - return 0; -#else for (i = 0; i < ncases; i++) { C = &cases[i]; - req = H5FD_s3comms_hrb_init_request(C->verb, - C->resource, - C->version); + req = H5FD_s3comms_hrb_init_request( + C->verb, + C->resource, + C->version); if (cases[i].ret_null == TRUE) { FAIL_IF( req != NULL ); - } else { + } + else { FAIL_IF( req == NULL ); JSVERIFY( S3COMMS_HRB_MAGIC, req->magic, NULL ) if (C->verb == NULL) { JSVERIFY_STR( "GET", req->verb, NULL ) - } else { + } + else { JSVERIFY_STR( req->verb, C->verb, NULL ) } JSVERIFY_STR( "HTTP/1.1", req->version, NULL ) @@ -823,7 +822,7 @@ test_hrb_init_request(void) FAIL_IF( NULL != req ); /* should annull pointer as well as free */ } - } /* for each testcase */ + } /* end for each testcase */ PASSED(); return 0; @@ -832,11 +831,10 @@ error: (void)H5FD_s3comms_hrb_destroy(&req); return -1; -#endif /* H5_HAVE_ROS3_VFD */ - -} /* test_hrb_init_request */ +} /* end test_hrb_init_request() */ + /*--------------------------------------------------------------------------- * * Function: test_hrb_node_set() @@ -853,7 +851,6 @@ error: static herr_t test_hrb_node_set(void) { -#ifdef H5_HAVE_ROS3_VFD /************************* * test-local structures * *************************/ @@ -1071,18 +1068,10 @@ test_hrb_node_set(void) }; unsigned testcases_count = 16; unsigned test_i = 0; - hrb_node_t *list = NULL; -#endif /* H5_HAVE_ROS3_VFD */ - TESTING("test_hrb_node_t"); + TESTING("hrb_node_t (test_hrb_node_set)"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD is not enabled"); - fflush(stdout); - return 0; -#else for (test_i = 0; test_i < testcases_count; test_i++) { const hrb_node_t *node = NULL; const testcase *test = &(cases[test_i]); @@ -1137,21 +1126,22 @@ test_hrb_node_set(void) FAIL_IF( SUCCEED != H5FD_s3comms_hrb_node_set(&list, list->name, NULL) ) } - } + } /* end for each testcase */ PASSED(); return 0; error: - while (list != NULL) + while (list != NULL) { (void)H5FD_s3comms_hrb_node_set(&list, list->name, NULL); + } return -1; -#endif /* H5_HAVE_ROS3_VFD */ -} /* test_hrb_node_t */ +} /* end test_hrb_node_t() */ + /*--------------------------------------------------------------------------- * * Function: test_HMAC_SHA256() @@ -1265,13 +1255,13 @@ test_HMAC_SHA256(void) dest); TEST_ERROR; } -#else +#else /* VERBOSE not defined */ /* simple pass/fail test */ JSVERIFY( 0, strncmp(cases[i].exp, dest, HDstrlen(cases[i].exp)), NULL); -#endif +#endif /* VERBOSE */ } free(dest); } @@ -1284,9 +1274,9 @@ error: return -1; #endif /* H5_HAVE_ROS3_VFD */ -} /* test_HMAC_SHA256 */ - +} /* end test_HMAC_SHA256() */ + /*---------------------------------------------------------------------------- * * Function: test_nlowercase() @@ -1307,7 +1297,6 @@ test_nlowercase(void) * test-local structures * *************************/ -#ifdef H5_HAVE_ROS3_VFD struct testcase { const char *in; size_t len; @@ -1338,16 +1327,9 @@ test_nlowercase(void) char *dest = NULL; int i = 0; int n_cases = 3; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("nlowercase"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD is not enabled"); - fflush(stdout); - return 0; -#else for (i = 0; i < n_cases; i++) { dest = (char *)HDmalloc(sizeof(char) * 16); @@ -1360,7 +1342,7 @@ test_nlowercase(void) JSVERIFY( 0, strncmp(dest, cases[i].exp, cases[i].len), NULL ) } free(dest); - } + } /* end for each testcase */ JSVERIFY( FAIL, H5FD_s3comms_nlowercase(NULL, @@ -1374,11 +1356,10 @@ test_nlowercase(void) error: free(dest); return -1; -#endif /* H5_HAVE_ROS3_VFD */ - -} /* test_nlowercase */ +} /* end test_nlowercase() */ + /*--------------------------------------------------------------------------- * * Function: test_parse_url() @@ -1399,7 +1380,6 @@ test_parse_url(void) * test-local structures * *************************/ -#ifdef H5_HAVE_ROS3_VFD typedef struct { const char *scheme; const char *host; @@ -1540,16 +1520,9 @@ test_parse_url(void) "non-decimal PORT (01a3)", }, }; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("url-parsing functionality"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD is not enabled"); - fflush(stdout); - return 0; -#else /********* * TESTS * *********/ @@ -1564,7 +1537,8 @@ test_parse_url(void) if (cases[i].exp_ret == FAIL) { /* on FAIL, `purl` should be untouched--remains NULL */ FAIL_UNLESS( purl == NULL ) - } else { + } + else { /* on SUCCEED, `purl` should be set */ FAIL_IF( purl == NULL ) @@ -1612,7 +1586,7 @@ test_parse_url(void) } else { FAIL_UNLESS( NULL == purl->query ) } - } /* if parse-url return SUCCEED/FAIL */ + } /* end if parse-url return SUCCEED/FAIL */ /* per-test cleanup * well-behaved, even if `purl` is NULL @@ -1620,7 +1594,7 @@ test_parse_url(void) FAIL_IF( FAIL == H5FD_s3comms_free_purl(purl) ) purl = NULL; - } /* for each testcase */ + } /* end for each testcase */ PASSED(); return 0; @@ -1632,11 +1606,10 @@ error: (void)H5FD_s3comms_free_purl(purl); return -1; -#endif /* H5_HAVE_ROS3_VFD */ - -} /* test_parse_url */ +} /* end test_parse_url() */ + /*--------------------------------------------------------------------------- * * Function: test_percent_encode_char() @@ -1662,7 +1635,6 @@ test_percent_encode_char(void) * test-local structures * *************************/ -#ifdef H5_HAVE_ROS3_VFD struct testcase { const char c; const char *exp; @@ -1692,16 +1664,9 @@ test_percent_encode_char(void) size_t dest_len = 0; int i = 0; int n_cases = 5; -#endif /* H5_HAVE_ROS3_VFD */ TESTING("percent encode characters"); -#ifndef H5_HAVE_ROS3_VFD - SKIPPED(); - puts(" ROS3 VFD is not enabled"); - fflush(stdout); - return 0; -#else for (i = 0; i < n_cases; i++) { JSVERIFY( SUCCEED, H5FD_s3comms_percent_encode_char( @@ -1726,11 +1691,9 @@ test_percent_encode_char(void) error: return -1; -#endif /* H5_HAVE_ROS3_VFD */ - -} /* test_percent_encode_char */ - +} /* end test_percent_encode_char() */ + /*--------------------------------------------------------------------------- * Function: test_s3r_open() * @@ -1798,9 +1761,9 @@ error: return -1; #endif /* H5_HAVE_ROS3_VFD */ -} /* test_s3r_get_filesize */ - +} /* end test_s3r_get_filesize() */ + /*--------------------------------------------------------------------------- * Function: test_s3r_open() * @@ -2039,9 +2002,9 @@ error: return -1; #endif /* H5_HAVE_ROS3_VFD */ -} /* test_s3r_open */ - +} /* end test_s3r_open() */ + /*--------------------------------------------------------------------------- * * Function: test_s3r_read() @@ -2265,7 +2228,7 @@ error: #undef S3COMMS_TEST_BUFFER_SIZE #endif /* H5_HAVE_ROS3_VFD */ -} /* test_s3r_read */ +} /* end test_s3r_read() */ /*--------------------------------------------------------------------------- @@ -2404,9 +2367,9 @@ error: return -1; #endif /* H5_HAVE_ROS3_VFD */ -} /* test_signing_key */ - +} /* end test_signing_key() */ + /*--------------------------------------------------------------------------- * * Function: test_tostringtosign() @@ -2476,9 +2439,9 @@ error : return -1; #endif /* H5_HAVE_ROS3_VFD */ -} /* test_tostringtosign */ - +} /* end test_tostringtosign() */ + /*---------------------------------------------------------------------------- * * Function: test_trim() @@ -2563,7 +2526,7 @@ test_trim(void) } free(str); str = NULL; - } + } /* end for each testcase */ JSVERIFY( SUCCEED, H5FD_s3comms_trim(dest, NULL, 3, &dest_len), "should not fail when trimming a null string" ); @@ -2582,13 +2545,14 @@ test_trim(void) return 0; error: - if (str != NULL) + if (str != NULL) { free(str); + } return -1; -} /* test_trim */ - +} /* end test_trim() */ + /*---------------------------------------------------------------------------- * * Function: test_uriencode() @@ -2685,7 +2649,7 @@ test_uriencode(void) free(dest); dest = NULL; - } + } /* end for each testcase */ /*************** * ERROR CASES * @@ -2713,11 +2677,11 @@ error: } return -1; -} /* test_uriencode */ - +} /* end test_uriencode() */ + /*------------------------------------------------------------------------- * Function: main() * @@ -2774,7 +2738,8 @@ main(void) if (bucket_url_env == NULL || bucket_url_env[0] == '\0') { HDprintf("WARNING: S3 bucket url is not defined in enviornment " \ "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n"); - } else { + } + else { HDstrncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE); s3_test_bucket_defined = TRUE; } @@ -2799,12 +2764,12 @@ main(void) nerrors += test_s3r_get_filesize() < 0 ? 1 : 0; nerrors += test_s3r_read() < 0 ? 1 : 0; - if(nerrors) { + if (nerrors) { HDprintf("***** %d S3comms TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); return 1; - } /* end if */ + } HDprintf("All S3comms tests passed.\n"); diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index 347c189..5df701a 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -24,6 +24,7 @@ static int doxml = 0; static int useschema = 1; static const char *xml_dtd_uri = NULL; +#ifdef H5_HAVE_ROS3_VFD static H5FD_ros3_fapl_t ros3_fa = { 1, /* version */ false, /* authenticate */ @@ -31,7 +32,9 @@ static H5FD_ros3_fapl_t ros3_fa = { "", /* access key id */ "", /* secret access key */ }; +#endif /* H5_HAVE_ROS3_VFD */ +#ifdef H5_HAVE_LIBHDFS static H5FD_hdfs_fapl_t hdfs_fa = { 1, /* fapl version */ "localhost", /* namenode name */ @@ -40,6 +43,7 @@ static H5FD_hdfs_fapl_t hdfs_fa = { "", /* user name */ 2048, /* stream buffer size */ }; +#endif /* H5_HAVE_LIBHDFS */ /* module-scoped variables for XML option */ #define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd" diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index 2364f23..7e2a431 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -2612,6 +2612,7 @@ main(int argc, const char *argv[]) int err_exit = 0; hid_t fapl_id = H5P_DEFAULT; +#ifdef H5_HAVE_ROS3_VFD /* default "anonymous" s3 configuration */ H5FD_ros3_fapl_t ros3_fa = { 1, /* fapl version */ @@ -2620,7 +2621,9 @@ main(int argc, const char *argv[]) "", /* access key id */ "", /* secret access key */ }; +#endif /* H5_HVAE_ROS3_VFD */ +#ifdef H5_HAVE_LIBHDFS /* "default" HDFS configuration */ H5FD_hdfs_fapl_t hdfs_fa = { 1, /* fapl version */ @@ -2630,6 +2633,7 @@ main(int argc, const char *argv[]) "", /* user name */ 2048, /* stream buffer size */ }; +#endif /* H5_HAVE_LIBHDFS */ h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index dd03f18..75d325a 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -120,6 +120,7 @@ typedef struct iter_t { static const char *drivername = ""; +#ifdef H5_HAVE_ROS3_VFD /* default "anonymous" s3 configuration */ static H5FD_ros3_fapl_t ros3_fa = { @@ -129,7 +130,9 @@ static H5FD_ros3_fapl_t ros3_fa = { "", /* access key id */ "", /* secret access key */ }; +#endif /* H5_HAVE_ROS3_VFD */ +#ifdef H5_HAVE_LIBHDFS /* default HDFS access configuration */ static H5FD_hdfs_fapl_t hdfs_fa = { @@ -140,6 +143,7 @@ static H5FD_hdfs_fapl_t hdfs_fa = { "", /* user name */ 2048, /* stream buffer size */ }; +#endif /* H5_HAVE_LIBHDFS */ static int display_all = TRUE; @@ -544,8 +548,8 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "attribute_stats() failed"); /* Get storage info */ - if((storage = H5Dget_storage_size(did)) < 0) - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_storage_size() failed"); + /* Failure 0 indistinguishable from no-data-stored 0 */ + storage = H5Dget_storage_size(did); /* Gather layout statistics */ if((dcpl = H5Dget_create_plist(did)) < 0) |