diff options
author | Jacob Smith <jake.smith@hdfgroup.org> | 2019-08-14 20:36:45 (GMT) |
---|---|---|
committer | Jacob Smith <jake.smith@hdfgroup.org> | 2019-08-14 20:36:45 (GMT) |
commit | 2e5cd3d2b2d953420252c80ab5e32ac95ac0c5b3 (patch) | |
tree | a79af758bc878d55a958bedae77d919e03a9516f /src | |
parent | 2b72832d97496290f74ccaffcda41c3e596f8f49 (diff) | |
download | hdf5-2e5cd3d2b2d953420252c80ab5e32ac95ac0c5b3.zip hdf5-2e5cd3d2b2d953420252c80ab5e32ac95ac0c5b3.tar.gz hdf5-2e5cd3d2b2d953420252c80ab5e32ac95ac0c5b3.tar.bz2 |
Fix compiler warnings with diabled ROS3 and HDFS VFDs.
Fix misuse of [HD]strncpy and [HD]snprintf.
Minor formatting changes.
Fix test cleanup issue with java groups example.
Diffstat (limited to 'src')
-rw-r--r-- | src/H5FDhdfs.c | 484 | ||||
-rw-r--r-- | src/H5FDros3.c | 392 | ||||
-rw-r--r-- | src/H5FDs3comms.c | 863 | ||||
-rw-r--r-- | src/H5FDs3comms.h | 2 |
4 files changed, 1032 insertions, 709 deletions
diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c index e3e11b2..83d0202 100644 --- a/src/H5FDhdfs.c +++ b/src/H5FDhdfs.c @@ -130,6 +130,8 @@ typedef struct { #endif /* HDFS_STATS */ +#ifdef H5_HAVE_LIBHDFS + /* "unique" identifier for `hdfs_t` structures. * Randomly generated by unweighted dice rolls. */ @@ -143,7 +145,7 @@ typedef struct { * Purpose: * * Contain/retain information associated with a file hosted on Hadoop - * Distributed File System (HDFS). Instantiated and populated via + * Distributed File System (HDFS). Instantiated and populated via * `H5FD_hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`. * * @@ -179,14 +181,11 @@ typedef struct { */ typedef struct { unsigned long magic; -#ifdef H5_HAVE_LIBHDFS hdfsFS filesystem; hdfsFileInfo *fileinfo; - hdfsFile file; -#endif + hdfsFile file; } hdfs_t; -#ifdef H5_HAVE_LIBHDFS /*-------------------------------------------------------------------------- * Function: H5FD_hdfs_handle_open @@ -274,7 +273,7 @@ H5FD_hdfs_handle_open( "hdfsGetPathInfo failed") } handle->file = hdfsOpenFile( - handle->filesystem, + handle->filesystem, path, O_RDONLY, stream_buffer_size, @@ -292,18 +291,18 @@ done: /* error; clean up */ HDassert(handle->magic == HDFS_HDFST_MAGIC); handle->magic++; - if (handle->file != NULL) { + if (handle->file != NULL) { if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close hdfs file handle") } } - if (handle->fileinfo != NULL) { + if (handle->fileinfo != NULL) { hdfsFreeFileInfo(handle->fileinfo, 1); } - if (handle->filesystem != NULL) { + if (handle->filesystem != NULL) { if (FAIL == (hdfsDisconnect(handle->filesystem))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to disconnect from hdfs") } } @@ -351,18 +350,18 @@ H5FD_hdfs_handle_close(hdfs_t *handle) } handle->magic++; - if (handle->file != NULL) { + if (handle->file != NULL) { if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close hdfs file handle") } } - if (handle->fileinfo != NULL) { + if (handle->fileinfo != NULL) { hdfsFreeFileInfo(handle->fileinfo, 1); } - if (handle->filesystem != NULL) { + if (handle->filesystem != NULL) { if (FAIL == (hdfsDisconnect(handle->filesystem))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to disconnect hdfs file system") } } @@ -373,6 +372,7 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_close() */ + #endif /* H5_HAVE_LIBHDFS */ @@ -382,9 +382,9 @@ done: * * Purpose: * - * H5FD_hdfs_t is a structure used to store all information needed to + * H5FD_hdfs_t is a structure used to store all information needed to * maintain R/O access to a single HDF5 file in an HDFS file system. - * This structure is created when such a file is "opened" and + * This structure is created when such a file is "opened" and * discarded when it is "closed". * * @@ -396,7 +396,7 @@ done: * * `fa` (H5FD_hdfs_fapl_t) * - * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data + * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data * needed to "open" the HDF5 file. * * `eoa` (haddr_t) @@ -405,9 +405,9 @@ done: * equal the file size. * * `hdfs_handle` (hdfs_t *) - * + * * Instance of HDFS Request handle associated with the target resource. - * Responsible for communicating with remote host and presenting file + * Responsible for communicating with remote host and presenting file * contents as indistinguishable from a file on the local filesystem. * * *** present only if HDFS_SATS is flagged to enable stats collection *** @@ -456,10 +456,13 @@ typedef struct H5FD_hdfs_t { * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' * is too large to be represented by the second argument * of the file seek function. + * Only included if HDFS code should compile. * */ #define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#ifdef H5_HAVE_LIBHDFS #define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#endif /* H5_HAVE_LIBHDFS */ /* Prototypes */ static herr_t H5FD_hdfs_term(void); @@ -474,13 +477,13 @@ static herr_t H5FD_hdfs_query(const H5FD_t *_f1, unsigned long *flags); static haddr_t H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t type); static herr_t H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); static haddr_t H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, +static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); -static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, void *buf); -static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, const void *buf); -static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, +static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD_hdfs_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD_hdfs_unlock(H5FD_t *_file); @@ -521,8 +524,10 @@ static const H5FD_class_t H5FD_hdfs_g = { H5FD_FLMAP_DICHOTOMY /* fl_map */ }; +#ifdef H5_HAVE_LIBHDFS /* Declare a free list to manage the H5FD_hdfs_t struct */ H5FL_DEFINE_STATIC(H5FD_hdfs_t); +#endif /* H5_HAVE_LIBHDFS */ /*------------------------------------------------------------------------- @@ -545,7 +550,7 @@ H5FD__init_package(void) FUNC_ENTER_STATIC if (H5FD_hdfs_init() < 0) { - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize hdfs VFD") } @@ -643,8 +648,8 @@ H5FD_hdfs_term(void) * Function: H5Pset_fapl_hdfs * * Purpose: Modify the file access property list to use the H5FD_HDFS - * driver defined in this source file. All driver specfic - * properties are passed in as a pointer to a suitably + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably * initialized instance of H5FD_hdfs_fapl_t * * Return: SUCCEED/FAIL @@ -658,7 +663,7 @@ H5FD_hdfs_term(void) *------------------------------------------------------------------------- */ herr_t -H5Pset_fapl_hdfs(hid_t fapl_id, +H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa) { H5P_genplist_t *plist = NULL; /* Property list pointer */ @@ -674,7 +679,7 @@ H5Pset_fapl_hdfs(hid_t fapl_id, #endif plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); - if (plist == NULL) { + if (plist == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, \ "not a file access property list") } @@ -699,12 +704,12 @@ done: * contains internally consistant data. Return SUCCEED if so, * and FAIL otherwise. * - * Note the difference between internally consistant and - * correct. As we will have to try to access the target + * Note the difference between internally consistant and + * correct. As we will have to try to access the target * object to determine whether the supplied data is correct, * we will settle for internal consistancy at this point * - * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally + * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally * consistant data, FAIL otherwise. * * Programmer: Jacob Smith @@ -761,7 +766,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_fapl_hdfs(hid_t fapl_id, +H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out) { const H5FD_hdfs_fapl_t *fa = NULL; @@ -832,7 +837,7 @@ H5FD_hdfs_fapl_get(H5FD_t *_file) fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t)); if (fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") } @@ -878,7 +883,7 @@ H5FD_hdfs_fapl_copy(const void *_old_fa) new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t)); if (new_fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") } @@ -916,7 +921,7 @@ H5FD_hdfs_fapl_free(void *_fa) FUNC_ENTER_NOAPI_NOINIT_NOERR - HDassert(fa != NULL); /* sanity check */ + HDassert(fa != NULL); /* sanity check */ H5MM_xfree(fa); @@ -994,11 +999,11 @@ done: * Create and/or opens a file as an HDF5 file. * * Any flag except H5F_ACC_RDONLY will cause an error. - * + * * Return: * - * Success: A pointer to a new file data structure. - * The public fields will be initialized by the caller, which is + * Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is * always H5FD_open(). * * Failure: NULL @@ -1010,26 +1015,21 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS static H5FD_t * H5FD_hdfs_open( - const char *path, - unsigned flags, - hid_t fapl_id, + const char *path, + unsigned flags, + hid_t fapl_id, haddr_t maxaddr) { H5FD_t *ret_value = NULL; -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = NULL; hdfs_t *handle = NULL; H5FD_hdfs_fapl_t fa; -#endif FUNC_ENTER_NOAPI_NOINIT -#ifndef H5_HAVE_LIBHDFS - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, - "Illegal open of unsupported virtual file (hdfs)"); -#else #if HDFS_DEBUG HDfprintf(stdout, "H5FD_hdfs_open() called.\n"); #endif /* HDFS_DEBUG */ @@ -1078,11 +1078,11 @@ H5FD_hdfs_open( HDassert(handle->magic == HDFS_HDFST_MAGIC); - /* create new file struct + /* create new file struct */ file = H5FL_CALLOC(H5FD_hdfs_t); if (file == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") } file->hdfs_handle = handle; @@ -1090,33 +1090,53 @@ H5FD_hdfs_open( #if HDFS_STATS if (FAIL == hdfs_reset_stats(file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics") } #endif /* HDFS_STATS */ ret_value = (H5FD_t*)file; -#endif /* H5_HAVE_LIBHDFS */ done: -#ifdef H5_HAVE_LIBHDFS if (ret_value == NULL) { - if (handle != NULL) { + if (handle != NULL) { if (FAIL == H5FD_hdfs_handle_close(handle)) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close HDFS file handle") } } if (file != NULL) { file = H5FL_FREE(H5FD_hdfs_t, file); } - } /* if null return value (error) */ -#endif /* H5_HAVE_LIBHDFS */ + } /* end if null return value (error) */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_open() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static H5FD_t * +H5FD_hdfs_open( + const char H5_ATTR_UNUSED *path, + unsigned H5_ATTR_UNUSED flags, + hid_t H5_ATTR_UNUSED fapl_id, + haddr_t H5_ATTR_UNUSED maxaddr) +{ + H5FD_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, + "Illegal open of unsupported virtual file (hdfs)"); +done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_open() */ +#endif /* H5_HAVE_LIBHDFS */ + #if HDFS_STATS /*---------------------------------------------------------------------------- @@ -1124,7 +1144,7 @@ done: * Function: hdfs_fprint_stats() * * Purpose: - * + * * Tabulate and pretty-print statistics for this virtual file. * * Should be called upon file close. @@ -1145,17 +1165,17 @@ done: * - max (largest size read) * 2. tabulation of "bins", sepraring reads into exponentially-larger * ranges of size. - * - columns for number of reads, total bytes, and average size, with + * - columns for number of reads, total bytes, and average size, with * separate sub-colums for raw- and metadata reads. * - each row represents one bin, identified by the top of its range - * + * * Bin ranges can be modified with pound-defines at the top of this file. * * Bins without any reads in their bounds are not printed. * * An "overflow" bin is also present, to catch "big" reads. * - * Output for all bins (and range ceiling and average size report) + * Output for all bins (and range ceiling and average size report) * is divied by powers of 1024. By corollary, four digits before the decimal * is valid. * @@ -1216,36 +1236,6 @@ hdfs_fprint_stats( "hdfs handle has invalid magic") } - /* TODO: See what libhdfs exposes to us. */ - -#if 0 - if (file->s3r_handle->purl == NULL) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "parsed url structure cannot be null") - } - purl = file->s3r_handle->purl; - - /****************** - * PRINT FILENAME * - ******************/ - - HDfprintf(stream, "stats for %s://%s", purl->scheme, purl->host); - if (purl->port != NULL && purl->port[0] != '\0') { - HDfprintf(stream, ":%s", purl->port); - } - if (purl->query != NULL && purl->query[0] != '\0') { - if (purl->path != NULL && purl->path[0] != '\0') { - HDfprintf(stream, "/%s", purl->path); - } else { - HDfprintf(stream, "/"); - } - HDfprintf(stream, "?%s", purl->query); - } else if (purl->path != NULL && purl->path[0] != '\0') { - HDfprintf(stream, "/%s", purl->path); - } - HDfprintf(stream, "\n"); -#endif - /******************* * AGGREGATE STATS * *******************/ @@ -1254,18 +1244,22 @@ hdfs_fprint_stats( const hdfs_statsbin *r = &file->raw[i]; const hdfs_statsbin *m = &file->meta[i]; - if (m->min < min_meta) min_meta = m->min; - if (r->min < min_raw) min_raw = r->min; - if (m->max > max_meta) max_meta = m->max; - if (r->max > max_raw) max_raw = r->max; + if (m->min < min_meta) { min_meta = m->min; } + if (r->min < min_raw) { min_raw = r->min; } + if (m->max > max_meta) { max_meta = m->max; } + if (r->max > max_raw) { max_raw = r->max; } count_raw += r->count; count_meta += m->count; bytes_raw += r->bytes; bytes_meta += m->bytes; } - if (count_raw > 0) average_raw = (double)bytes_raw / (double)count_raw; - if (count_meta > 0) average_meta = (double)bytes_meta / (double)count_meta; + if (count_raw > 0) { + average_raw = (double)bytes_raw / (double)count_raw; + } + if (count_meta > 0) { + average_meta = (double)bytes_meta / (double)count_meta; + } /****************** * PRINT OVERVIEW * @@ -1342,9 +1336,9 @@ hdfs_fprint_stats( * PRINT INDIVIDUAL BIN STATS * ******************************/ - HDfprintf(stream, + HDfprintf(stream, "BINS # of reads total bytes average size\n"); - HDfprintf(stream, + HDfprintf(stream, " up-to meta raw meta raw meta raw\n"); for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { @@ -1414,7 +1408,7 @@ hdfs_fprint_stats( HDassert(suffix_i < sizeof(suffixes)); HDfprintf( - stream, + stream, " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, suffixes[suffix_i], /* bin ceiling */ m->count, /* metadata reads */ @@ -1428,7 +1422,6 @@ hdfs_fprint_stats( done: FUNC_LEAVE_NOAPI(ret_value); - } /* hdfs_fprint_stats */ #endif /* HDFS_STATS */ @@ -1442,7 +1435,7 @@ done: * Close an HDF5 file. * * Return: - * + * * SUCCEED/FAIL * * Programmer: Jacob Smith @@ -1452,35 +1445,31 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_close(H5FD_t *_file) { herr_t ret_value = SUCCEED; -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; -#endif FUNC_ENTER_NOAPI_NOINIT -#ifndef H5_HAVE_LIBHDFS - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, - "Illegal close of unsupported Virtual File (hdfs)") -#else #if HDFS_DEBUG HDfprintf(stdout, "H5FD_hdfs_close() called.\n"); #endif - /* Sanity checks + /* Sanity checks */ HDassert(file != NULL); HDassert(file->hdfs_handle != NULL); HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); - /* Close the underlying request handle + /* Close the underlying request handle */ - if (file->hdfs_handle != NULL) { + if (file->hdfs_handle != NULL) { if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) { - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close HDFS file handle") } } @@ -1488,21 +1477,37 @@ H5FD_hdfs_close(H5FD_t *_file) #if HDFS_STATS /* TODO: mechanism to re-target stats printout */ if (FAIL == hdfs_fprint_stats(stdout, file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics") } #endif /* HDFS_STATS */ - /* Release the file info + /* Release the file info */ file = H5FL_FREE(H5FD_hdfs_t, file); -#endif /* H5_HAVE_LIBHDFS */ done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_close() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "Illegal close of unsupported Virtual File (hdfs)") + +done: + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_close() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1527,21 +1532,21 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static int -H5FD_hdfs_cmp(const H5FD_t *_f1, - const H5FD_t *_f2) +H5FD_hdfs_cmp( + const H5FD_t *_f1, + const H5FD_t *_f2) { int ret_value = 0; -#ifdef H5_HAVE_LIBHDFS const H5FD_hdfs_t *f1 = (const H5FD_hdfs_t *)_f1; const H5FD_hdfs_t *f2 = (const H5FD_hdfs_t *)_f2; hdfsFileInfo *finfo1 = NULL; hdfsFileInfo *finfo2 = NULL; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR -#ifdef H5_HAVE_LIBHDFS #if HDFS_DEBUG HDfprintf(stdout, "H5FD_hdfs_cmp() called.\n"); #endif /* HDFS_DEBUG */ @@ -1556,23 +1561,37 @@ H5FD_hdfs_cmp(const H5FD_t *_f1, HDassert(finfo1 != NULL); HDassert(finfo2 != NULL); - if (finfo1->mKind != finfo2->mKind) HGOTO_DONE(-1); - if (finfo1->mName != finfo2->mName) HGOTO_DONE(-1); - if (finfo1->mLastMod != finfo2->mLastMod) HGOTO_DONE(-1); - if (finfo1->mSize != finfo2->mSize) HGOTO_DONE(-1); - if (finfo1->mReplication != finfo2->mReplication) HGOTO_DONE(-1); - if (finfo1->mBlockSize != finfo2->mBlockSize) HGOTO_DONE(-1); - if (strcmp(finfo1->mOwner, finfo2->mOwner)) HGOTO_DONE(-1); - if (strcmp(finfo1->mGroup, finfo2->mGroup)) HGOTO_DONE(-1); - if (finfo1->mPermissions != finfo2->mPermissions) HGOTO_DONE(-1); - if (finfo1->mLastAccess != finfo2->mLastAccess) HGOTO_DONE(-1); -#endif /* H5_HAVE_LIBHDFS */ + if (finfo1->mKind != finfo2->mKind) { HGOTO_DONE(-1); } + if (finfo1->mName != finfo2->mName) { HGOTO_DONE(-1); } + if (finfo1->mLastMod != finfo2->mLastMod) { HGOTO_DONE(-1); } + if (finfo1->mSize != finfo2->mSize) { HGOTO_DONE(-1); } + if (finfo1->mReplication != finfo2->mReplication) { HGOTO_DONE(-1); } + if (finfo1->mBlockSize != finfo2->mBlockSize) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mOwner, finfo2->mOwner)) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mGroup, finfo2->mGroup)) { HGOTO_DONE(-1); } + if (finfo1->mPermissions != finfo2->mPermissions) { HGOTO_DONE(-1); } + if (finfo1->mLastAccess != finfo2->mLastAccess) { HGOTO_DONE(-1); } done: FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_cmp() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static int +H5FD_hdfs_cmp( + const H5FD_t H5_ATTR_UNUSED *_f1, + const H5FD_t H5_ATTR_UNUSED *_f2) +{ + int ret_value = 0; + FUNC_ENTER_NOAPI_NOINIT_NOERR + + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_cmp() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * Function: H5FD_hdfs_query @@ -1580,7 +1599,7 @@ done: * Purpose: Set the flags that this VFL driver is capable of supporting. * (listed in H5FDpublic.h) * - * Note that since the HDFS VFD is read only, most flags + * Note that since the HDFS VFD is read only, most flags * are irrelevant. * * The term "set" is highly misleading... @@ -1595,7 +1614,7 @@ done: */ static herr_t H5FD_hdfs_query( - const H5FD_t H5_ATTR_UNUSED *_file, + const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) /* out variable */ { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1606,11 +1625,10 @@ H5FD_hdfs_query( if (flags) { *flags = 0; - *flags |= H5FD_FEAT_DATA_SIEVE; + *flags |= H5FD_FEAT_DATA_SIEVE; } FUNC_LEAVE_NOAPI(SUCCEED) - } /* H5FD_hdfs_query() */ @@ -1635,14 +1653,14 @@ H5FD_hdfs_query( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static haddr_t H5FD_hdfs_get_eoa( - const H5FD_t *_file, + const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { -#ifdef H5_HAVE_LIBHDFS const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1650,14 +1668,27 @@ H5FD_hdfs_get_eoa( HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); #endif -#ifdef H5_HAVE_LIBHDFS FUNC_LEAVE_NOAPI(file->eoa) -#else - FUNC_LEAVE_NOAPI(0) -#endif /* H5_HAVE_LIBHDFS */ +} /* end H5FD_hdfs_get_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eoa( + const H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(0) } /* end H5FD_hdfs_get_eoa() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1678,15 +1709,15 @@ H5FD_hdfs_get_eoa( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_set_eoa( - H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, + H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) { -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1694,16 +1725,30 @@ H5FD_hdfs_set_eoa( HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); #endif -#ifdef H5_HAVE_LIBHDFS file->eoa = addr; FUNC_LEAVE_NOAPI(SUCCEED) -#else - FUNC_LEAVE_NOAPI(FAIL) -#endif /* H5_HAVE_LIBHDFS */ +} /* H5FD_hdfs_set_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_set_eoa( + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t H5_ATTR_UNUSED addr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(FAIL) } /* H5FD_hdfs_set_eoa() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1715,7 +1760,7 @@ H5FD_hdfs_set_eoa( * * Return: * - * EOF: the first address past the end of the "file", either the + * EOF: the first address past the end of the "file", either the * filesystem file or the HDF5 file. * * Programmer: Jacob Smith @@ -1723,14 +1768,14 @@ H5FD_hdfs_set_eoa( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static haddr_t H5FD_hdfs_get_eof( - const H5FD_t *_file, + const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { -#ifdef H5_HAVE_LIBHDFS const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; -#endif /* H5_HAVE_LIBHDFS */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1738,17 +1783,30 @@ H5FD_hdfs_get_eof( HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); #endif -#ifdef H5_HAVE_LIBHDFS HDassert(file->hdfs_handle != NULL); HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); FUNC_LEAVE_NOAPI((size_t) file->hdfs_handle->fileinfo->mSize) -#else - FUNC_LEAVE_NOAPI((size_t)0) -#endif /* H5_HAVE_LIBHDFS */ +} /* end H5FD_hdfs_get_eof() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eof( + const H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); +#endif + + FUNC_LEAVE_NOAPI((size_t)0) } /* end H5FD_hdfs_get_eof() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * @@ -1769,16 +1827,16 @@ H5FD_hdfs_get_eof( * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_get_handle( - H5FD_t *_file, - hid_t H5_ATTR_UNUSED fapl, + H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, void **file_handle) { herr_t ret_value = SUCCEED; -#ifdef H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; -#endif FUNC_ENTER_NOAPI_NOINIT @@ -1786,29 +1844,48 @@ H5FD_hdfs_get_handle( HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); #endif /* HDFS_DEBUG */ -#ifdef H5_HAVE_LIBHDFS if (!file_handle) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") } *file_handle = file->hdfs_handle; -#else + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_get_handle() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_get_handle( + H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED fapl, + void H5_ATTR_UNUSED **file_handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); +#endif /* HDFS_DEBUG */ + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "Illegal get-handle of unsupported virtual file (hdfs)"); -#endif /* H5_HAVE_LIBHDFS */ done: FUNC_LEAVE_NOAPI(ret_value) - } /* end H5FD_hdfs_get_handle() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * * Function: H5FD_hdfs_read() * - * Purpose: + * Purpose: * * Reads SIZE bytes of data from FILE beginning at address ADDR * into buffer BUF according to data transfer properties in DXPL_ID. @@ -1828,20 +1905,20 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_LIBHDFS + static herr_t H5FD_hdfs_read( - H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, + H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, /* start offset */ size_t size, /* length of read */ void *buf) /* out */ { herr_t ret_value = SUCCEED; -#if H5_HAVE_LIBHDFS H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; size_t filesize = 0; -#endif /* H5_HAVE_LIBHDFS */ #if HDFS_STATS /* working variables for storing stats */ hdfs_statsbin *bin = NULL; @@ -1854,10 +1931,6 @@ H5FD_hdfs_read( HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); #endif /* HDFS_DEBUG */ -#ifndef H5_HAVE_LIBHDFS - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, - "Illegal get-handle of unsupported virtual file (hdfs)"); -#else HDassert(file != NULL); HDassert(file->hdfs_handle != NULL); HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); @@ -1894,37 +1967,64 @@ H5FD_hdfs_read( ? &file->raw[bin_i] : &file->meta[bin_i]; - /* Store collected stats in appropriate bin + /* Store collected stats in appropriate bin */ if (bin->count == 0) { bin->min = size; bin->max = size; - } else { - if (size < bin->min) bin->min = size; - if (size > bin->max) bin->max = size; + } + else { + if (size < bin->min) { bin->min = size; } + if (size > bin->max) { bin->max = size; } } bin->count++; bin->bytes += (unsigned long long)size; #endif /* HDFS_STATS */ -#endif /* H5_HAVE_LIBHDFS */ done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_read() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_read( + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, + void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); +#endif /* HDFS_DEBUG */ + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "Illegal get-handle of unsupported virtual file (hdfs)"); +done: + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_read() */ +#endif /* H5_HAVE_LIBHDFS */ + /*------------------------------------------------------------------------- * * Function: H5FD_hdfs_write() * - * Purpose: + * Purpose: * * Write bytes to file. * UNSUPPORTED IN READ-ONLY HDFS VFD. * - * Return: + * Return: * * FAIL (Not possible with Read-Only S3 file.) * @@ -1937,11 +2037,11 @@ done: */ static herr_t H5FD_hdfs_write( - H5FD_t H5_ATTR_UNUSED *_file, - H5FD_mem_t H5_ATTR_UNUSED type, + H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, - haddr_t H5_ATTR_UNUSED addr, - size_t H5_ATTR_UNUSED size, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf) { herr_t ret_value = FAIL; @@ -1957,7 +2057,6 @@ H5FD_hdfs_write( done: FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_hdfs_write() */ @@ -1985,8 +2084,8 @@ done: */ static herr_t H5FD_hdfs_truncate( - H5FD_t H5_ATTR_UNUSED *_file, - hid_t H5_ATTR_UNUSED dxpl_id, + H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) { herr_t ret_value = SUCCEED; @@ -2002,7 +2101,6 @@ H5FD_hdfs_truncate( done: FUNC_LEAVE_NOAPI(ret_value) - } /* end H5FD_hdfs_truncate() */ @@ -2031,12 +2129,11 @@ done: */ static herr_t H5FD_hdfs_lock( - H5FD_t H5_ATTR_UNUSED *_file, + H5FD_t H5_ATTR_UNUSED *_file, hbool_t H5_ATTR_UNUSED rw) { FUNC_ENTER_NOAPI_NOINIT_NOERR FUNC_LEAVE_NOAPI(SUCCEED) - } /* end H5FD_hdfs_lock() */ @@ -2065,6 +2162,5 @@ H5FD_hdfs_unlock(H5FD_t H5_ATTR_UNUSED *_file) { FUNC_ENTER_NOAPI_NOINIT_NOERR FUNC_LEAVE_NOAPI(SUCCEED) - } /* end H5FD_hdfs_unlock() */ diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 86642aa..18f1f91 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -11,12 +11,12 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Read-Only S3 Virtual File Driver (VFD) + * Read-Only S3 Virtual File Driver (VFD) * * Programmer: Jacob Smith <jake.smith@hdfgroup.org> * 2017-10-13 * - * Purpose: + * Purpose: * * Provide read-only access to files hosted on Amazon's S3 service. * Relies on "s3comms" utility layer to implement the AWS REST API. @@ -137,9 +137,9 @@ typedef struct { * * Purpose: * - * H5FD_ros3_t is a structure used to store all information needed to + * H5FD_ros3_t is a structure used to store all information needed to * maintain R/O access to a single HDF5 file that has been stored as a - * S3 object. This structure is created when such a file is "opened" and + * S3 object. This structure is created when such a file is "opened" and * discarded when it is "closed". * * Presents an S3 object as a file to the HDF5 library. @@ -154,7 +154,7 @@ typedef struct { * * `fa` (H5FD_ros3_fapl_t) * - * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data + * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data * needed to "open" the HDF5 file. * * `eoa` (haddr_t) @@ -163,9 +163,9 @@ typedef struct { * equal the file size. * * `s3r_handle` (s3r_t *) - * + * * Instance of S3 Request handle associated with the target resource. - * Responsible for communicating with remote host and presenting file + * Responsible for communicating with remote host and presenting file * contents as indistinguishable from a file on the local filesystem. * * *** present only if ROS3_SATS is flagged to enable stats collection *** @@ -211,10 +211,13 @@ typedef struct H5FD_ros3_t { * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' * is too large to be represented by the second argument * of the file seek function. + * Only included if it may be used -- ROS3 VFD is enabled. * */ #define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#ifdef H5_HAVE_ROS3_VFD #define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#endif /* H5_HAVE_ROS3_VFD */ /* Prototypes */ static herr_t H5FD_ros3_term(void); @@ -229,13 +232,13 @@ static herr_t H5FD_ros3_query(const H5FD_t *_f1, unsigned long *flags); static haddr_t H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t type); static herr_t H5FD_ros3_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); static haddr_t H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, +static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); -static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, void *buf); -static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, +static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, const void *buf); -static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, +static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD_ros3_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD_ros3_unlock(H5FD_t *_file); @@ -276,8 +279,10 @@ static const H5FD_class_t H5FD_ros3_g = { H5FD_FLMAP_DICHOTOMY /* fl_map */ }; +#ifdef H5_HAVE_ROS3_VFD /* Declare a free list to manage the H5FD_ros3_t struct */ H5FL_DEFINE_STATIC(H5FD_ros3_t); +#endif /* H5_HAVE_ROS3_VFD */ /*------------------------------------------------------------------------- @@ -300,14 +305,14 @@ H5FD__init_package(void) FUNC_ENTER_STATIC if (H5FD_ros3_init() < 0) { - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize ros3 VFD") } done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ +} /* end H5FD__init_package() */ /*------------------------------------------------------------------------- @@ -330,7 +335,7 @@ done: hid_t H5FD_ros3_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + hid_t ret_value = H5I_INVALID_HID; FUNC_ENTER_NOAPI(FAIL) @@ -388,6 +393,7 @@ H5FD_ros3_term(void) H5FD_ROS3_g = 0; FUNC_LEAVE_NOAPI(SUCCEED) + } /* end H5FD_ros3_term() */ @@ -395,8 +401,8 @@ H5FD_ros3_term(void) * Function: H5Pset_fapl_ros3 * * Purpose: Modify the file access property list to use the H5FD_ROS3 - * driver defined in this source file. All driver specfic - * properties are passed in as a pointer to a suitably + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably * initialized instance of H5FD_ros3_fapl_t * * Return: SUCCEED/FAIL @@ -407,14 +413,12 @@ H5FD_ros3_term(void) *------------------------------------------------------------------------- */ herr_t -H5Pset_fapl_ros3(hid_t fapl_id, +H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa) { H5P_genplist_t *plist = NULL; /* Property list pointer */ herr_t ret_value = FAIL; - - FUNC_ENTER_API(FAIL) H5TRACE2("e", "i*x", fapl_id, fa); @@ -425,21 +429,21 @@ H5Pset_fapl_ros3(hid_t fapl_id, #endif plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); - if (plist == NULL) { + if (plist == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, \ "not a file access property list") } - if (FAIL == H5FD_ros3_validate_config(fa)) + if (FAIL == H5FD_ros3_validate_config(fa)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config") + } ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa); done: - FUNC_LEAVE_API(ret_value) -} /* H5Pset_fapl_ros3() */ +} /* end H5Pset_fapl_ros3() */ /*------------------------------------------------------------------------- @@ -449,12 +453,12 @@ done: * contains internally consistant data. Return SUCCEED if so, * and FAIL otherwise. * - * Note the difference between internally consistant and - * correct. As we will have to try to access the target + * Note the difference between internally consistant and + * correct. As we will have to try to access the target * object to determine whether the supplied data is correct, * we will settle for internal consistancy at this point * - * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally + * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally * consistant data, FAIL otherwise. * * Programmer: Jacob Smith @@ -494,7 +498,7 @@ H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_ros3_validate_config() */ +} /* end H5FD_ros3_validate_config() */ /*------------------------------------------------------------------------- @@ -515,12 +519,12 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_fapl_ros3(hid_t fapl_id, +H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out) { - const H5FD_ros3_fapl_t *fa; - H5P_genplist_t *plist = NULL; /* Property list pointer */ - herr_t ret_value = SUCCEED; /* Return value */ + const H5FD_ros3_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_API(FAIL) H5TRACE2("e", "i*x", fapl_id, fa_out); @@ -529,8 +533,9 @@ H5Pget_fapl_ros3(hid_t fapl_id, HDfprintf(stdout, "H5Pget_fapl_ros3() called.\n"); #endif - if (fa_out == NULL) + if (fa_out == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL") + } plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); if (plist == NULL) { @@ -552,7 +557,7 @@ H5Pget_fapl_ros3(hid_t fapl_id, done: FUNC_LEAVE_API(ret_value) -} /* H5Pget_fapl_ros3() */ +} /* end H5Pget_fapl_ros3() */ /*------------------------------------------------------------------------- @@ -583,7 +588,7 @@ H5FD_ros3_fapl_get(H5FD_t *_file) fa = (H5FD_ros3_fapl_t *)H5MM_calloc(sizeof(H5FD_ros3_fapl_t)); if (fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") } @@ -595,12 +600,13 @@ H5FD_ros3_fapl_get(H5FD_t *_file) done: if (ret_value == NULL) { - if (fa != NULL) - H5MM_xfree(fa); + if (fa != NULL) { + H5MM_xfree(fa); + } } FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_ros3_fapl_get() */ +} /* end H5FD_ros3_fapl_get() */ /*------------------------------------------------------------------------- @@ -624,13 +630,13 @@ H5FD_ros3_fapl_copy(const void *_old_fa) { const H5FD_ros3_fapl_t *old_fa = (const H5FD_ros3_fapl_t*)_old_fa; H5FD_ros3_fapl_t *new_fa = NULL; - void *ret_value = NULL; /* Return value */ + void *ret_value = NULL; FUNC_ENTER_NOAPI_NOINIT new_fa = (H5FD_ros3_fapl_t *)H5MM_malloc(sizeof(H5FD_ros3_fapl_t)); if (new_fa == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); } @@ -639,12 +645,13 @@ H5FD_ros3_fapl_copy(const void *_old_fa) done: if (ret_value == NULL) { - if (new_fa != NULL) + if (new_fa != NULL) { H5MM_xfree(new_fa); + } } FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_ros3_fapl_copy() */ +} /* end H5FD_ros3_fapl_copy() */ /*------------------------------------------------------------------------- @@ -668,13 +675,13 @@ H5FD_ros3_fapl_free(void *_fa) FUNC_ENTER_NOAPI_NOINIT_NOERR - HDassert(fa != NULL); /* sanity check */ + HDassert(fa != NULL); /* sanity check */ H5MM_xfree(fa); FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FD_ros3_fapl_free() */ +} /* end H5FD_ros3_fapl_free() */ #if ROS3_STATS @@ -707,7 +714,6 @@ ros3_reset_stats(H5FD_ros3_t *file) unsigned i = 0; herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT #if ROS3_DEBUG @@ -734,7 +740,8 @@ ros3_reset_stats(H5FD_ros3_t *file) done: FUNC_LEAVE_NOAPI(ret_value); -} /* ros3_reset_stats */ +} /* end ros3_reset_stats() */ + #endif /* ROS3_STATS */ @@ -747,10 +754,10 @@ done: * Create and/or opens a file as an HDF5 file. * * Any flag except H5F_ACC_RDONLY will cause an error. - * + * * Name (as received from `H5FD_open()`) must conform to web url: * NAME :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ] - * HTTP :: "http" [ "s" ] + * HTTP :: "http" [ "s" ] * DOMAIN :: e.g., "mybucket.host.org" * PORT :: ":" <number> (e.g., ":9000" ) * URI :: <string> (e.g., "path/to/resource.hd5" ) @@ -758,8 +765,8 @@ done: * * Return: * - * Success: A pointer to a new file data structure. - * The public fields will be initialized by the caller, which is + * Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is * always H5FD_open(). * * Failure: NULL @@ -771,32 +778,31 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + static H5FD_t * -H5FD_ros3_open(const char *url, - unsigned flags, - hid_t fapl_id, - haddr_t maxaddr) +H5FD_ros3_open( + const char *url, + unsigned flags, + hid_t fapl_id, + haddr_t maxaddr) { -#ifdef H5_HAVE_ROS3_VFD H5FD_ros3_t *file = NULL; struct tm *now = NULL; char iso8601now[ISO8601_SIZE]; unsigned char signing_key[SHA256_DIGEST_LENGTH]; s3r_t *handle = NULL; H5FD_ros3_fapl_t fa; -#endif H5FD_t *ret_value = NULL; FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_open() called.\n"); #endif - /* Sanity check on file offsets */ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); @@ -837,7 +843,7 @@ H5FD_ros3_open(const char *url, if (FAIL == H5FD_s3comms_signing_key(signing_key, (const char *)fa.secret_key, (const char *)fa.aws_region, - (const char *)iso8601now) ) + (const char *)iso8601now) ) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key") @@ -854,17 +860,17 @@ H5FD_ros3_open(const char *url, if (handle == NULL) { /* If we want to check CURL's say on the matter in a controlled - * fashion, this is the place to do it, but would need to make a + * fashion, this is the place to do it, but would need to make a * few minor changes to s3comms `s3r_t` and `s3r_read()`. - */ + */ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open"); } - /* create new file struct + /* create new file struct */ file = H5FL_CALLOC(H5FD_ros3_t); if (file == NULL) { - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") } @@ -873,7 +879,7 @@ H5FD_ros3_open(const char *url, #if ROS3_STATS if (FAIL == ros3_reset_stats(file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics") } #endif /* ROS3_STATS */ @@ -882,9 +888,9 @@ H5FD_ros3_open(const char *url, done: if (ret_value == NULL) { - if (handle != NULL) { + if (handle != NULL) { if (FAIL == H5FD_s3comms_s3r_close(handle)) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle") } } @@ -892,13 +898,28 @@ done: file = H5FL_FREE(H5FD_ros3_t, file); } curl_global_cleanup(); /* early cleanup because open failed */ - } /* if null return value (error) */ -#endif /* H5_HAVE_ROS3_VFD */ + } /* end if null return value (error) */ FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_ros3_open() */ + +#else /* H5_HAVE_ROS3_VFD not defined */ + +static H5FD_t * +H5FD_ros3_open( + const char H5_ATTR_UNUSED *url, + unsigned H5_ATTR_UNUSED flags, + hid_t H5_ATTR_UNUSED fapl_id, + haddr_t H5_ATTR_UNUSED maxaddr) +{ + H5FD_t *ret_value = NULL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_ros3_open() */ +#endif /* H5_HAVE_ROS3_VFD */ + #if ROS3_STATS /*---------------------------------------------------------------------------- @@ -906,7 +927,7 @@ done: * Function: ros3_fprint_stats() * * Purpose: - * + * * Tabulate and pretty-print statistics for this virtual file. * * Should be called upon file close. @@ -927,17 +948,17 @@ done: * - max (largest size read) * 2. tabulation of "bins", sepraring reads into exponentially-larger * ranges of size. - * - columns for number of reads, total bytes, and average size, with + * - columns for number of reads, total bytes, and average size, with * separate sub-colums for raw- and metadata reads. * - each row represents one bin, identified by the top of its range - * + * * Bin ranges can be modified with pound-defines at the top of this file. * * Bins without any reads in their bounds are not printed. * * An "overflow" bin is also present, to catch "big" reads. * - * Output for all bins (and range ceiling and average size report) + * Output for all bins (and range ceiling and average size report) * is divied by powers of 1024. By corollary, four digits before the decimal * is valid. * @@ -1050,7 +1071,7 @@ ros3_fprint_stats(FILE *stream, HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", bytes_raw + bytes_meta, bytes_meta, bytes_raw); - if (count_raw + count_meta == 0) + if (count_raw + count_meta == 0) goto done; /************************* @@ -1109,9 +1130,9 @@ ros3_fprint_stats(FILE *stream, * PRINT INDIVIDUAL BIN STATS * ******************************/ - HDfprintf(stream, + HDfprintf(stream, "BINS # of reads total bytes average size\n"); - HDfprintf(stream, + HDfprintf(stream, " up-to meta raw meta raw meta raw\n"); for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { @@ -1129,7 +1150,7 @@ ros3_fprint_stats(FILE *stream, m = &file->meta[i]; r = &file->raw[i]; - if (r->count == 0 && m->count == 0) + if (r->count == 0 && m->count == 0) continue; range_end = ros3_stats_boundaries[i]; @@ -1172,7 +1193,7 @@ ros3_fprint_stats(FILE *stream, re_dub /= 1024.0; HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, + HDfprintf(stream, " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, suffixes[suffix_i], /* bin ceiling */ m->count, /* metadata reads */ @@ -1187,7 +1208,7 @@ ros3_fprint_stats(FILE *stream, done: FUNC_LEAVE_NOAPI(ret_value); - + } /* ros3_fprint_stats */ #endif /* ROS3_STATS */ @@ -1201,7 +1222,7 @@ done: * Close an HDF5 file. * * Return: - * + * * SUCCEED/FAIL * * Programmer: Jacob Smith @@ -1211,57 +1232,63 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + static herr_t -H5FD_ros3_close(H5FD_t *_file) +H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file) { -#ifdef H5_HAVE_ROS3_VFD H5FD_ros3_t *file = (H5FD_ros3_t *)_file; herr_t ret_value = SUCCEED; -#else - herr_t ret_value = FAIL; -#endif - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_close() called.\n"); #endif - /* Sanity checks + /* Sanity checks */ HDassert(file != NULL); HDassert(file->s3r_handle != NULL); - /* Close the underlying request handle + /* Close the underlying request handle */ if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) { - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle") } #if ROS3_STATS /* TODO: mechanism to re-target stats printout */ - if (FAIL == ros3_fprint_stats(stdout, file)) { - HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, + if (ros3_fprint_stats(stdout, file) == FAIL) { + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics") } #endif /* ROS3_STATS */ - /* Release the file info + /* Release the file info */ file = H5FL_FREE(H5FD_ros3_t, file); done: curl_global_cleanup(); /* cleanup to answer init on open */ -#endif /* H5_HAVE_ROS3_VFD */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_ros3_close() */ +#else /* H5_HAVE_ROS3_VFD not defined */ + +static herr_t +H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_ros3_close() */ + +#endif /* H5_HAVE_ROS3_VFD */ + /*------------------------------------------------------------------------- * @@ -1269,7 +1296,7 @@ done: * * Purpose: * - * Compares two files belonging to this driver using an arbitrary + * Compares two files belonging to this driver using an arbitrary * (but consistent) ordering: * * + url scheme @@ -1284,7 +1311,7 @@ done: * tl;dr -> check URL, check crentials * * Return: - * + * * - Equivalent: 0 * - Not Equivalent: -1 * @@ -1300,22 +1327,20 @@ done: * *------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + static int -H5FD_ros3_cmp(const H5FD_t *_f1, - const H5FD_t *_f2) +H5FD_ros3_cmp( + const H5FD_t *_f1, + const H5FD_t *_f2) { -#ifdef H5_HAVE_ROS3_VFD const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1; const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2; const parsed_url_t *purl1 = NULL; const parsed_url_t *purl2 = NULL; -#endif int ret_value = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR -#ifdef H5_HAVE_ROS3_VFD #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_cmp() called.\n"); @@ -1334,71 +1359,124 @@ H5FD_ros3_cmp(const H5FD_t *_f1, HDassert(purl2->host != NULL); /* URL: SCHEME */ - if (HDstrcmp(purl1->scheme, purl2->scheme)) HGOTO_DONE(-1); + if (HDstrcmp(purl1->scheme, purl2->scheme)) { + HGOTO_DONE(-1); + } /* URL: HOST */ - if (HDstrcmp(purl1->host, purl2->host)) HGOTO_DONE(-1); + if (HDstrcmp(purl1->host, purl2->host)) { + HGOTO_DONE(-1); + } /* URL: PORT */ if (purl1->port && purl2->port) { - if (HDstrcmp(purl1->port, purl2->port)) HGOTO_DONE(-1); - } else if (purl1->port) { + if (HDstrcmp(purl1->port, purl2->port)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->port) { HGOTO_DONE(-1); - } else if (purl2->port) { + } + else + if (purl2->port) { HGOTO_DONE(-1); } /* URL: PATH */ if (purl1->path && purl2->path) { - if (HDstrcmp(purl1->path, purl2->path)) HGOTO_DONE(-1); - } else if (purl1->path && !purl2->path) { + if (HDstrcmp(purl1->path, purl2->path)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->path && !purl2->path) { HGOTO_DONE(-1); - } else if (purl2->path && !purl1->path) { + } + else + if (purl2->path && !purl1->path) { HGOTO_DONE(-1); } /* URL: QUERY */ if (purl1->query && purl2->query) { - if (HDstrcmp(purl1->query, purl2->query)) HGOTO_DONE(-1); - } else if (purl1->query && !purl2->query) { + if (HDstrcmp(purl1->query, purl2->query)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->query && !purl2->query) { HGOTO_DONE(-1); - } else if (purl2->query && !purl1->query) { + } + else + if (purl2->query && !purl1->query) { HGOTO_DONE(-1); } /* FAPL: AWS_REGION */ if (f1->fa.aws_region[0] != '\0' && f1->fa.aws_region[0] != '\0') { - if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) HGOTO_DONE(-1); - } else if (f1->fa.aws_region[0] != '\0') { + if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.aws_region[0] != '\0') { HGOTO_DONE(-1); - } else if (f2->fa.aws_region[0] != '\0') { + } + else + if (f2->fa.aws_region[0] != '\0') { HGOTO_DONE(-1); } /* FAPL: SECRET_ID */ if (f1->fa.secret_id[0] != '\0' && f1->fa.secret_id[0] != '\0') { - if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) HGOTO_DONE(-1); - } else if (f1->fa.secret_id[0] != '\0') { + if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_id[0] != '\0') { HGOTO_DONE(-1); - } else if (f2->fa.secret_id[0] != '\0') { + } + else + if (f2->fa.secret_id[0] != '\0') { HGOTO_DONE(-1); } /* FAPL: SECRET_KEY */ if (f1->fa.secret_key[0] != '\0' && f1->fa.secret_key[0] != '\0') { - if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) HGOTO_DONE(-1); - } else if (f1->fa.secret_key[0] != '\0') { + if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_key[0] != '\0') { HGOTO_DONE(-1); - } else if (f2->fa.secret_key[0] != '\0') { + } + else + if (f2->fa.secret_key[0] != '\0') { HGOTO_DONE(-1); } -#endif /* H5_HAVE_ROS3_VFD */ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_ros3_cmp() */ +#else /* H5_HAVE_ROS3_VFD not defined */ + +static int +H5FD_ros3_cmp( + const H5FD_t H5_ATTR_UNUSED *_f1, + const H5FD_t H5_ATTR_UNUSED *_f2) +{ + int ret_value = 0; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_ros3_cmp() */ + +#endif /* H5_HAVE_ROS3_VFD */ + /*------------------------------------------------------------------------- * Function: H5FD_ros3_query @@ -1406,7 +1484,7 @@ done: * Purpose: Set the flags that this VFL driver is capable of supporting. * (listed in H5FDpublic.h) * - * Note that since the ROS3 VFD is read only, most flags + * Note that since the ROS3 VFD is read only, most flags * are irrelevant. * * The term "set" is highly misleading... @@ -1420,7 +1498,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, +H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out */) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1433,7 +1511,7 @@ H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, if (flags) { *flags = 0; /* OK to perform data sieving for faster raw data reads & writes */ - *flags |= H5FD_FEAT_DATA_SIEVE; + *flags |= H5FD_FEAT_DATA_SIEVE; } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) @@ -1463,7 +1541,7 @@ H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, *------------------------------------------------------------------------- */ static haddr_t -H5FD_ros3_get_eoa(const H5FD_t *_file, +H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; @@ -1499,8 +1577,8 @@ H5FD_ros3_get_eoa(const H5FD_t *_file, *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_set_eoa(H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, +H5FD_ros3_set_eoa(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) { H5FD_ros3_t *file = (H5FD_ros3_t *)_file; @@ -1528,7 +1606,7 @@ H5FD_ros3_set_eoa(H5FD_t *_file, * * Return: * - * EOF: the first address past the end of the "file", either the + * EOF: the first address past the end of the "file", either the * filesystem file or the HDF5 file. * * Programmer: Jacob Smith @@ -1537,7 +1615,7 @@ H5FD_ros3_set_eoa(H5FD_t *_file, *------------------------------------------------------------------------- */ static haddr_t -H5FD_ros3_get_eof(const H5FD_t *_file, +H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; @@ -1573,8 +1651,8 @@ H5FD_ros3_get_eof(const H5FD_t *_file, *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_get_handle(H5FD_t *_file, - hid_t H5_ATTR_UNUSED fapl, +H5FD_ros3_get_handle(H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, void **file_handle) { H5FD_ros3_t *file = (H5FD_ros3_t *)_file; @@ -1586,8 +1664,9 @@ H5FD_ros3_get_handle(H5FD_t *_file, HDfprintf(stdout, "H5FD_ros3_get_handle() called.\n"); #endif - if(!file_handle) + if (!file_handle) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") + } *file_handle = file->s3r_handle; @@ -1601,7 +1680,7 @@ done: * * Function: H5FD_ros3_read() * - * Purpose: + * Purpose * * Reads SIZE bytes of data from FILE beginning at address ADDR * into buffer BUF according to data transfer properties in DXPL_ID. @@ -1622,8 +1701,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_read(H5FD_t *_file, - H5FD_mem_t H5_ATTR_UNUSED type, +H5FD_ros3_read(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, /* start offset */ size_t size, /* length of read */ @@ -1631,13 +1710,13 @@ H5FD_ros3_read(H5FD_t *_file, { H5FD_ros3_t *file = (H5FD_ros3_t *)_file; size_t filesize = 0; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; #if ROS3_STATS /* working variables for storing stats */ ros3_statsbin *bin = NULL; unsigned bin_i = 0; #endif /* ROS3_STATS */ - + FUNC_ENTER_NOAPI_NOINIT @@ -1655,7 +1734,7 @@ H5FD_ros3_read(H5FD_t *_file, HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address") } - if (FAIL == H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) ) { + if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) { HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read") } @@ -1664,30 +1743,33 @@ H5FD_ros3_read(H5FD_t *_file, /* Find which "bin" this read fits in. Can be "overflow" bin. */ for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) { - if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) + if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) { break; + } } bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i]; - /* Store collected stats in appropriate bin + /* Store collected stats in appropriate bin */ if (bin->count == 0) { bin->min = size; bin->max = size; - } else { - if (size < bin->min) + } + else { + if (size < bin->min) { bin->min = size; - if (size > bin->max) + } + if (size > bin->max) { bin->max = size; + } } bin->count++; bin->bytes += (unsigned long long)size; #endif /* ROS3_STATS */ - done: FUNC_LEAVE_NOAPI(ret_value) @@ -1698,12 +1780,12 @@ done: * * Function: H5FD_ros3_write() * - * Purpose: + * Purpose: * * Write bytes to file. * UNSUPPORTED IN READ-ONLY ROS3 VFD. * - * Return: + * Return: * * FAIL (Not possible with Read-Only S3 file.) * @@ -1715,11 +1797,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file, - H5FD_mem_t H5_ATTR_UNUSED type, +H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, - haddr_t H5_ATTR_UNUSED addr, - size_t H5_ATTR_UNUSED size, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf) { herr_t ret_value = FAIL; @@ -1762,8 +1844,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, - hid_t H5_ATTR_UNUSED dxpl_id, +H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) { herr_t ret_value = SUCCEED; @@ -1807,7 +1889,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file, +H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file, hbool_t H5_ATTR_UNUSED rw) { FUNC_ENTER_NOAPI_NOINIT_NOERR diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 8c0515d..42b281f 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /***************************************************************************** - * Read-Only S3 Virtual File Driver (VFD) + * Read-Only S3 Virtual File Driver (VFD) * * Source for S3 Communications module * @@ -51,6 +51,8 @@ */ #define S3COMMS_DEBUG 0 +#ifdef H5_HAVE_ROS3_VFD + /* manipulate verbosity of CURL output * operates separately from S3COMMS_DEBUG * @@ -65,6 +67,8 @@ */ #define S3COMMS_MAX_RANGE_STRING_SIZE 128 +#endif /* H5_HAVE_ROS3_VFD */ + /******************/ /* Local Typedefs */ /******************/ @@ -111,7 +115,7 @@ herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle); /* Functions */ /*************/ - + /*---------------------------------------------------------------------------- * * Function: curlwritecallback() @@ -147,8 +151,9 @@ curlwritecallback(char *ptr, size_t product = (size * nmemb); size_t written = 0; - if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) + if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) { return written; + } if (size > 0) { HDmemcpy(&(sds->data[sds->size]), ptr, product); @@ -158,9 +163,9 @@ curlwritecallback(char *ptr, return written; -} /* curlwritecallback */ - +} /* end curlwritecallback() */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_hrb_node_set() @@ -223,9 +228,10 @@ curlwritecallback(char *ptr, *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_hrb_node_set(hrb_node_t **L, - const char *name, - const char *value) +H5FD_s3comms_hrb_node_set( + hrb_node_t **L, + const char *name, + const char *value) { size_t i = 0; char *valuecpy = NULL; @@ -238,8 +244,6 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, hbool_t is_looking = TRUE; herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -247,8 +251,9 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, HDprintf("NAME: %s\n", name); HDprintf("VALUE: %s\n", value); HDprintf("LIST:\n->"); - for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) + for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) { HDfprintf(stdout, "{%s}\n->", node_ptr->cat); + } HDprintf("(null)\n"); fflush(stdout); node_ptr = NULL; @@ -281,34 +286,37 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, * copies */ if (value != NULL) { - size_t valuelen = HDstrlen(value); - size_t catlen = namelen + valuelen + 2; /* HDstrlen(": ") -> +2 */ - int sprint_ret = 0; + int ret = 0; + size_t valuelen = HDstrlen(value); + size_t catlen = namelen + valuelen + 2; /* +2 from ": " */ + size_t catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */ + namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); if (namecpy == NULL) { HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for name copy.\n"); } - HDmemcpy(namecpy, name, namelen + 1); + HDmemcpy(namecpy, name, (namelen + 1)); valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1)); if (valuecpy == NULL) { HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for value copy.\n"); } - HDmemcpy(valuecpy, value, valuelen + 1); + HDmemcpy(valuecpy, value, (valuelen + 1)); - nvcat = (char *)H5MM_malloc(sizeof(char) * (catlen + 1)); + nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite); if (nvcat == NULL) { HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for concatenated string.\n"); } - sprint_ret = HDsnprintf(nvcat, (catlen + 1), "%s: %s", name, value); - if (sprint_ret <= 0) + ret = HDsnprintf(nvcat, catwrite, "%s: %s", name, value); + if (ret < 0 || (size_t)ret > catlen) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "error while concatenating `%s: %s", name, value); - HDassert( catlen == (size_t)sprint_ret ); + "cannot concatenate `%s: %s", name, value); + } + HDassert( catlen == HDstrlen(nvcat) ); /* create new_node, should we need it */ @@ -334,7 +342,8 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove node from empty list"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("CREATE NEW\n"); fflush(stdout); #endif @@ -406,7 +415,8 @@ HDprintf("FREEING POINTER\n"); fflush(stdout); HDprintf("FREEING WORKING LOWERNAME\n"); fflush(stdout); #endif H5MM_xfree(lowername); lowername = NULL; - } else { + } + else { #if S3COMMS_DEBUG HDprintf("MODIFY HEAD\n"); fflush(stdout); #endif @@ -428,14 +438,17 @@ HDprintf("MODIFY HEAD\n"); fflush(stdout); H5MM_xfree(new_node); new_node = NULL; } - } else if (strcmp(lowername, node_ptr->lowername) < 0) { + } + else + if (strcmp(lowername, node_ptr->lowername) < 0) { is_looking = FALSE; if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove a node 'before' head"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("PREPEND NEW HEAD\n"); fflush(stdout); #endif @@ -464,7 +477,8 @@ HDprintf("PREPEND NEW HEAD\n"); fflush(stdout); if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove absent node"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("APPEND A NODE\n"); fflush(stdout); #endif @@ -479,14 +493,17 @@ HDprintf("APPEND A NODE\n"); fflush(stdout); new_node->cat = nvcat; node_ptr->next = new_node; } - } else if (strcmp(lowername, node_ptr->next->lowername) < 0) { + } + else + if (strcmp(lowername, node_ptr->next->lowername) < 0) { is_looking = FALSE; if (value == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "trying to remove absent node"); - } else { + } + else { #if S3COMMS_DEBUG HDprintf("INSERT A NODE\n"); fflush(stdout); #endif @@ -502,7 +519,9 @@ HDprintf("INSERT A NODE\n"); fflush(stdout); new_node->next = node_ptr->next; node_ptr->next = new_node; } - } else if (strcmp(lowername, node_ptr->next->lowername) == 0) { + } + else + if (strcmp(lowername, node_ptr->next->lowername) == 0) { is_looking = FALSE; @@ -528,7 +547,8 @@ HDprintf("REMOVE A NODE\n"); fflush(stdout); H5MM_xfree(lowername); lowername = NULL; - } else { + } + else { #if S3COMMS_DEBUG HDprintf("MODIFY A NODE\n"); fflush(stdout); #endif @@ -552,23 +572,24 @@ HDprintf("MODIFY A NODE\n"); fflush(stdout); node_ptr->value = valuecpy; node_ptr->cat = nvcat; } - } else { + } + else { /**************** * KEEP LOOKING * ****************/ node_ptr = node_ptr->next; } - } + } /* end while is_looking */ done: if (ret_value == FAIL) { /* clean up */ - if (nvcat != NULL) H5MM_xfree(nvcat); - if (namecpy != NULL) H5MM_xfree(namecpy); - if (lowername != NULL) H5MM_xfree(lowername); - if (valuecpy != NULL) H5MM_xfree(valuecpy); + if (nvcat != NULL) { H5MM_xfree(nvcat); } + if (namecpy != NULL) { H5MM_xfree(namecpy); } + if (lowername != NULL) { H5MM_xfree(lowername); } + if (valuecpy != NULL) { H5MM_xfree(valuecpy); } if (new_node != NULL) { HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC ); new_node->magic += 1ul; @@ -577,11 +598,9 @@ done: } FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_hrb_node_set() */ -} /* H5FD_s3comms_hrb_node_set */ - - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_hrb_destroy() @@ -637,8 +656,6 @@ H5FD_s3comms_hrb_destroy(hrb_t **_buf) hrb_t *buf = NULL; herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -658,12 +675,11 @@ H5FD_s3comms_hrb_destroy(hrb_t **_buf) buf->magic += 1ul; H5MM_xfree(buf); *_buf = NULL; - } + } /* end if `_buf` has some value */ done: FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD_s3comms_hrb_destroy */ +} /* end H5FD_s3comms_hrb_destroy() */ /*---------------------------------------------------------------------------- @@ -728,8 +744,6 @@ H5FD_s3comms_hrb_init_request(const char *_verb, char *vrsn = NULL; size_t vrsnlen = 0; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -743,11 +757,13 @@ H5FD_s3comms_hrb_init_request(const char *_verb, /* populate valid NULLs with defaults */ - if (_verb == NULL) + if (_verb == NULL) { _verb = "GET"; + } - if (_http_version == NULL) + if (_http_version == NULL) { _http_version = "HTTP/1.1"; + } /* malloc space for and prepare structure */ @@ -765,28 +781,26 @@ H5FD_s3comms_hrb_init_request(const char *_verb, /* malloc and copy strings for the structure */ + reslen = HDstrlen(_resource); + if (_resource[0] == '/') { - reslen = HDstrlen(_resource) + 1; - res = (char *)H5MM_malloc(sizeof(char) * reslen); + res = (char *)H5MM_malloc(sizeof(char) * (reslen+1)); if (res == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string"); } - HDstrncpy(res, _resource, reslen); - } else { - int sprint_ret = 0; - reslen = HDstrlen(_resource) + 2; - res = (char *)H5MM_malloc(sizeof(char) * reslen); + HDmemcpy(res, _resource, (reslen+1)); + } + else { + res = (char *)H5MM_malloc(sizeof(char) * (reslen+2)); if (res == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, - "no space for resource string"); + "no space for resource string"); } - sprint_ret = HDsnprintf(res, reslen, "/%s", _resource); - if (sprint_ret <= 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, - "errro while appending resource string %s", _resource); - HDassert( (reslen - 1) == (size_t)sprint_ret ); - } /* start resource string with '/' */ + *res = '/'; + HDmemcpy((&res[1]), _resource, (reslen+1)); + HDassert( (reslen+1) == HDstrlen(res) ); + } /* end if (else resource string not starting with '/') */ verblen = HDstrlen(_verb) + 1; verb = (char *)H5MM_malloc(sizeof(char) * verblen); @@ -815,7 +829,6 @@ H5FD_s3comms_hrb_init_request(const char *_verb, ret_value = request; done: - /* if there is an error, clean up after ourselves */ if (ret_value == NULL) { @@ -826,8 +839,7 @@ done: } FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD_s3comms_hrb_init_request */ +} /* end H5FD_s3comms_hrb_init_request() */ @@ -835,7 +847,60 @@ done: * S3R FUNCTIONS ****************************************************************************/ +#ifndef H5_HAVE_ROS3_VFD +/* No-op, auto-fail, unused-variable implementations if ROS3 is not enabled */ + +herr_t +H5FD_s3comms_s3r_close(s3r_t H5_ATTR_UNUSED *handle) +{ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_s3r_close() */ + +size_t +H5FD_s3comms_s3r_get_filesize(s3r_t H5_ATTR_UNUSED *handle) +{ + size_t ret_value = 0; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_s3r_get_filesize() */ + +herr_t +H5FD_s3comms_s3r_getsize(s3r_t H5_ATTR_UNUSED *handle) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_s3r_getsize() */ + +s3r_t * +H5FD_s3comms_s3r_open( + const char H5_ATTR_UNUSED *url, + const char H5_ATTR_UNUSED *region, + const char H5_ATTR_UNUSED *id, + const unsigned char H5_ATTR_UNUSED *signing_key) +{ + s3r_t *ret_value = NULL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_s3r_open() */ +herr_t +H5FD_s3comms_s3r_read( + s3r_t H5_ATTR_UNUSED *handle, + haddr_t H5_ATTR_UNUSED offset, + size_t H5_ATTR_UNUSED len, + void H5_ATTR_UNUSED *dest) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_s3r_read() */ + +#else + + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_close() @@ -876,11 +941,8 @@ H5FD_s3comms_s3r_close(s3r_t *handle) { herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_s3r_close.\n"); #endif @@ -910,14 +972,11 @@ H5FD_s3comms_s3r_close(s3r_t *handle) H5MM_xfree(handle); -#endif /* H5_HAVE_ROS3_VFD */ - done: FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_s3comms_s3r_close */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_get_filesize() @@ -941,22 +1000,20 @@ done: *---------------------------------------------------------------------------- */ size_t -H5FD_s3comms_s3r_get_filesize(s3r_t *handle) { - +H5FD_s3comms_s3r_get_filesize(s3r_t *handle) +{ size_t ret_value = 0; FUNC_ENTER_NOAPI_NOINIT_NOERR -#ifdef H5_HAVE_ROS3_VFD - if (handle != NULL) + if (handle != NULL) { ret_value = handle->filesize; -#endif /* H5_HAVE_ROS3_VFD */ + } FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_s3comms_s3r_get_filesize */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_getsize() @@ -1143,7 +1200,7 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t\n"); } - if (content_length == 0 || + if (content_length == 0 || errno == ERANGE) /* errno set by strtoumax*/ { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -1187,7 +1244,7 @@ done: } /* H5FD_s3comms_s3r_getsize */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_open() @@ -1463,7 +1520,7 @@ done: } /* H5FD_s3comms_s3r_open */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_s3r_read() @@ -1621,7 +1678,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, if (len > 0) { rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ - S3COMMS_MAX_RANGE_STRING_SIZE ); + (S3COMMS_MAX_RANGE_STRING_SIZE+1) ); if (rangebytesstr == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "could not malloc range format string.\n"); @@ -1631,12 +1688,12 @@ H5FD_s3comms_s3r_read(s3r_t *handle, "bytes="H5_PRINTF_HADDR_FMT"-"H5_PRINTF_HADDR_FMT, offset, offset + len - 1); - if (ret == 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format HTTP Range value"); } else if (offset > 0) { rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ - S3COMMS_MAX_RANGE_STRING_SIZE); + (S3COMMS_MAX_RANGE_STRING_SIZE+1)); if (rangebytesstr == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "could not malloc range format string.\n"); @@ -1645,7 +1702,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, (S3COMMS_MAX_RANGE_STRING_SIZE), "bytes="H5_PRINTF_HADDR_FMT"-", offset); - if (ret == 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format HTTP Range value"); } @@ -1679,7 +1736,7 @@ H5FD_s3comms_s3r_read(s3r_t *handle, } else { /* authenticate request */ - char authorization[512]; + char authorization[512+1]; /* 512 := approximate max length... * 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request," * "SignedHeaders=,Signature=")> @@ -1689,10 +1746,10 @@ H5FD_s3comms_s3r_read(s3r_t *handle, * + 20 <max? len(region)> * + 128 <max? len(signed_headers)> */ - char buffer1[512]; /* -> Canonical Request -> Signature */ - char buffer2[256]; /* -> String To Sign -> Credential */ + char buffer1[512+1]; /* -> Canonical Request -> Signature */ + char buffer2[256+1]; /* -> String To Sign -> Credential */ char iso8601now[ISO8601_SIZE]; - char signed_headers[48]; + char signed_headers[48+1]; /* should be large enough for nominal listing: * "host;range;x-amz-content-sha256;x-amz-date" * + '\0', with "range;" possibly absent @@ -1821,8 +1878,11 @@ H5FD_s3comms_s3r_read(s3r_t *handle, /**** COMPUTE AUTHORIZATION ****/ if (FAIL == /* buffer1 -> canonical request */ - H5FD_s3comms_aws_canonical_request(buffer1, + H5FD_s3comms_aws_canonical_request( + buffer1, + 512, signed_headers, + 48, request) ) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -1858,23 +1918,25 @@ H5FD_s3comms_s3r_read(s3r_t *handle, HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 credential string"); - ret = HDsnprintf(authorization, + ret = HDsnprintf( + authorization, 512, "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", buffer2, signed_headers, buffer1); - if (ret == 0 || ret >= 512) + if (ret <= 0 || ret >= 512) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to format aws4 authorization string"); + } /* append authorization header to http request buffer */ - if (FAIL == - H5FD_s3comms_hrb_node_set( - &headers, - "Authorization", - (const char *)authorization) ) + if (H5FD_s3comms_hrb_node_set( + &headers, + "Authorization", + (const char *)authorization) + == FAIL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to set Authorization header") @@ -1914,17 +1976,18 @@ H5FD_s3comms_s3r_read(s3r_t *handle, /* finally, set http headers in curl handle */ - if (CURLE_OK != - curl_easy_setopt(curlh, - CURLOPT_HTTPHEADER, - curlheaders) ) + if (curl_easy_setopt( + curlh, + CURLOPT_HTTPHEADER, + curlheaders) + != CURLE_OK) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "error while setting CURL option " "(CURLOPT_HTTPHEADER). (placeholder flags)"); } - } /* if should authenticate (info provided) */ + } /* end if should authenticate (info provided) */ /******************* * PERFORM REQUEST * @@ -1983,14 +2046,16 @@ H5FD_s3comms_s3r_read(s3r_t *handle, HDfprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n"); if (sds == NULL) { HDfprintf(stderr, "sds is NULL!\n"); - } else { + } + else { HDfprintf(stderr, "sds: 0x%lx\n", (long long)sds); HDfprintf(stderr, "sds->size: %d\n", (int)sds->size); if (len > sds->size) { HDfprintf(stderr, "buffer overwrite\n"); } } - } else { + } + else { HDfprintf(stderr, "performed on entire file\n"); } #endif @@ -2028,29 +2093,30 @@ done: if (curlh != NULL) { /* clear any Range */ - if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) { HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_RANGE") + } /* clear headers */ - if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) { HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot unset CURLOPT_HTTPHEADER") + } } #endif /* H5_HAVE_ROS3_VFD */ FUNC_LEAVE_NOAPI(ret_value); - } /* H5FD_s3comms_s3r_read */ - +#endif /* H5_HAVE_ROS3_VFD */ /**************************************************************************** * MISCELLANEOUS FUNCTIONS ****************************************************************************/ - + /*---------------------------------------------------------------------------- * * Function: gmnow() @@ -2079,14 +2145,14 @@ gmnow(void) struct tm *ret_value = NULL; /* Doctor assert, checks against error in time() */ - if ( (time_t)(-1) != time(now_ptr) ) + if ( (time_t)(-1) != time(now_ptr) ) { ret_value = gmtime(now_ptr); + } HDassert( ret_value != NULL ); return ret_value; - -} /* gmnow */ +} /* end gmnow() */ /*---------------------------------------------------------------------------- @@ -2131,16 +2197,23 @@ gmnow(void) *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, - char *signed_headers_dest, - hrb_t *http_request) +H5FD_s3comms_aws_canonical_request( + char *canonical_request_dest, + int _cr_size, + char *signed_headers_dest, + int _sh_size, + hrb_t *http_request) { hrb_node_t *node = NULL; const char *query_params = ""; /* unused at present */ herr_t ret_value = SUCCEED; - int ret = 0; /* return value of HDsnprintf */ - size_t len = 0; /* working string length variable */ - char tmpstr[256]; + int ret = 0; + size_t cr_size = (size_t)_cr_size; + size_t sh_size = (size_t)_sh_size; + size_t cr_len = 0; /* working length of canonical request str */ + size_t sh_len = 0; /* working length of signed headers str */ + char tmpstr[256+1]; + tmpstr[256] = 0; /* terminating NULL */ /* "query params" refers to the optional element in the URL, e.g. * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J @@ -2152,8 +2225,6 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, * VFD use-cases. */ - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -2178,55 +2249,71 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, /* HTTP verb, resource path, and query string lines */ - len = (HDstrlen(http_request->verb) + + cr_len = (HDstrlen(http_request->verb) + HDstrlen(http_request->resource) + HDstrlen(query_params) + - 3 ); - ret = HDsnprintf(canonical_request_dest, - len + 1, - "%s\n%s\n%s\n", - http_request->verb, - http_request->resource, - query_params); - if (ret == 0 || (size_t)ret > len) + (size_t)3); /* three newline chars */ + if (cr_len >= cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } + ret = HDsnprintf( /* TODO: compiler warning */ + canonical_request_dest, + (cr_size-1), + "%s\n%s\n%s\n", + http_request->verb, + http_request->resource, + query_params); + if (ret < 0 || (size_t)ret >= cr_size) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to compose canonical request first line"); + } /* write in canonical headers, building signed headers concurrently */ - node = http_request->first_header; /* assumed at first sorted */ + node = http_request->first_header; /* assumed sorted */ while (node != NULL) { - size_t join_len = 0; /* string len of joined header-value */ - - HDassert( node->magic == S3COMMS_HRB_NODE_MAGIC ); - - len = HDstrlen(node->lowername); - join_len = HDstrlen(node->value) + len + 2; /* +2 <- ":\n" */ - ret = HDsnprintf(tmpstr, - join_len + 1, /* +1 for null terminator */ - "%s:%s\n", - node->lowername, - node->value); - if (ret == 0 || (size_t)ret > join_len) + + HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC); + + ret = HDsnprintf( + tmpstr, + 256, + "%s:%s\n", + node->lowername, + node->value); + if (ret < 0 || ret >= 256) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to concatenate HTTP header %s:%s", node->lowername, node->value); + } + cr_len += HDstrlen(tmpstr); + if (cr_len + 1 > cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } strcat(canonical_request_dest, tmpstr); - len += 1; /* semicolon */ - ret = HDsnprintf(tmpstr, - len + 1, - "%s;", - node->lowername); - if (ret == 0 || (size_t)ret > len) + ret = HDsnprintf( + tmpstr, + 256, + "%s;", + node->lowername); + if (ret < 0 || ret >= 256) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "unable to append signed header %s", + "unable to append semicolon to lowername %s", node->lowername); + } + sh_len += HDstrlen(tmpstr); + if (sh_len + 1 > sh_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in signed headers"); + } strcat(signed_headers_dest, tmpstr); node = node->next; - } + } /* end while node is not NULL */ /* remove tailing ';' from signed headers sequence */ @@ -2243,10 +2330,9 @@ H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, done: FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_aws_canonical_request() */ -} /* H5FD_s3comms_aws_canonical_request */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_bytes_to_hex() @@ -2293,16 +2379,15 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_bytes_to_hex(char *dest, - const unsigned char *msg, - size_t msg_len, - hbool_t lowercase) +H5FD_s3comms_bytes_to_hex( + char *dest, + const unsigned char *msg, + size_t msg_len, + hbool_t lowercase) { size_t i = 0; herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -2319,24 +2404,23 @@ H5FD_s3comms_bytes_to_hex(char *dest, } for (i = 0; i < msg_len; i++) { - int chars_written = - HDsnprintf(&(dest[i * 2]), - 3, /* 'X', 'X', '\n' */ - (lowercase == TRUE) ? "%02x" - : "%02X", - msg[i]); - if (chars_written != 2) + int chars_written = HDsnprintf(&(dest[i * 2]), + 3, /* 'X', 'X', '\n' */ + (lowercase == TRUE) ? "%02x" : "%02X", + msg[i]); + if (chars_written != 2) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem while writing hex chars for %c", msg[i]); + } } done: FUNC_LEAVE_NOAPI(ret_value); -} /* H5FD_s3comms_bytes_to_hex */ - +} /* end H5FD_s3comms_bytes_to_hex() */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_free_purl() @@ -2369,19 +2453,19 @@ H5FD_s3comms_free_purl(parsed_url_t *purl) if (purl != NULL) { HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC ); - if (purl->scheme != NULL) H5MM_xfree(purl->scheme); - if (purl->host != NULL) H5MM_xfree(purl->host); - if (purl->port != NULL) H5MM_xfree(purl->port); - if (purl->path != NULL) H5MM_xfree(purl->path); - if (purl->query != NULL) H5MM_xfree(purl->query); + if (purl->scheme != NULL) { H5MM_xfree(purl->scheme); } + if (purl->host != NULL) { H5MM_xfree(purl->host); } + if (purl->port != NULL) { H5MM_xfree(purl->port); } + if (purl->path != NULL) { H5MM_xfree(purl->path); } + if (purl->query != NULL) { H5MM_xfree(purl->query); } purl->magic += 1ul; H5MM_xfree(purl); } FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FD_s3comms_free_purl */ - +} /* end H5FD_s3comms_free_purl() */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_HMAC_SHA256() @@ -2424,27 +2508,22 @@ H5FD_s3comms_free_purl(parsed_url_t *purl) * *---------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + herr_t -H5FD_s3comms_HMAC_SHA256(const unsigned char *key, - size_t key_len, - const char *msg, - size_t msg_len, - char *dest) +H5FD_s3comms_HMAC_SHA256( + const unsigned char *key, + size_t key_len, + const char *msg, + size_t msg_len, + char *dest) { -#ifdef H5_HAVE_ROS3_VFD unsigned char md[SHA256_DIGEST_LENGTH]; unsigned int md_len = SHA256_DIGEST_LENGTH; herr_t ret_value = SUCCEED; -#else - herr_t ret_value = FAIL; -#endif /* H5_HAVE_ROS3_VFD */ - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD - #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_HMAC_SHA256.\n"); #endif @@ -2462,24 +2541,39 @@ H5FD_s3comms_HMAC_SHA256(const unsigned char *key, md, &md_len); - if (FAIL == - H5FD_s3comms_bytes_to_hex(dest, - (const unsigned char *)md, - (size_t)md_len, - true)) + if (H5FD_s3comms_bytes_to_hex( + dest, + (const unsigned char *)md, + (size_t)md_len, + true) + == FAIL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not convert to hex string."); } -#endif /* H5_HAVE_ROS3_VFD */ - done: FUNC_LEAVE_NOAPI(ret_value); - } /* H5FD_s3comms_HMAC_SHA256 */ +#else /* H5_HAVE_ROS3_VFD not defined */ + +herr_t +H5FD_s3comms_HMAC_SHA256( + const unsigned char H5_ATTR_UNUSED *key, + size_t H5_ATTR_UNUSED key_len, + const char H5_ATTR_UNUSED *msg, + size_t H5_ATTR_UNUSED msg_len, + char H5_ATTR_UNUSED *dest) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_HMAC_SHA256() */ + +#endif /* H5_HAVE_ROS3_VFD */ + /*----------------------------------------------------------------------------- * * Function: H5FD__s3comms_load_aws_creds_from_file() @@ -2565,85 +2659,98 @@ H5FD__s3comms_load_aws_creds_from_file( #endif /* format target line for start of profile */ - if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) + if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format profile label") + } /* look for start of profile */ do { /* clear buffer */ - for (buffer_i=0; buffer_i < 128; buffer_i++) buffer[buffer_i] = 0; + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } line_buffer = fgets(line_buffer, 128, file); - if (line_buffer == NULL) /* reached end of file */ + if (line_buffer == NULL) { /* reached end of file */ goto done; + } } while (strncmp(line_buffer, profile_line, HDstrlen(profile_line))); /* extract credentials from lines */ do { - size_t setting_name_len = 0; - const char *setting_name = NULL; - char line_prefix[128]; - /* clear buffer */ - for (buffer_i=0; buffer_i < 128; buffer_i++) buffer[buffer_i] = 0; + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } /* collect a line from file */ line_buffer = fgets(line_buffer, 128, file); - if (line_buffer == NULL) + if (line_buffer == NULL) { goto done; /* end of file */ + } /* loop over names to see if line looks like assignment */ for (setting_i = 0; setting_i < setting_count; setting_i++) { + size_t setting_name_len = 0; + const char *setting_name = NULL; + char line_prefix[128]; + setting_name = setting_names[setting_i]; setting_name_len = HDstrlen(setting_name); - if (128 < HDsnprintf( - line_prefix, - setting_name_len+2, - "%s=", - setting_name)) + if (HDsnprintf(line_prefix, 128, "%s=", setting_name) < 0) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format line prefix") + } /* found a matching name? */ if (!HDstrncmp(line_buffer, line_prefix, setting_name_len + 1)) { found_setting = 1; /* skip NULL destination buffer */ - if (setting_pointers[setting_i] == NULL) + if (setting_pointers[setting_i] == NULL) { break; + } /* advance to end of name in string */ do { line_buffer++; } while (*line_buffer != 0 && *line_buffer != '='); - if (*line_buffer == 0 || *(line_buffer+1) == 0) + if (*line_buffer == 0 || *(line_buffer+1) == 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "incomplete assignment in file") + } line_buffer++; /* was pointing at '='; advance */ /* copy line buffer into out pointer */ - HDstrncpy(setting_pointers[setting_i], (const char *)line_buffer, - HDstrlen(line_buffer)); + if (HDstrncpy( + setting_pointers[setting_i], + (const char *)line_buffer, + HDstrlen(line_buffer)) + == NULL) + { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to copy line into pointer") + } /* "trim" tailing whitespace by replacing with null terminator*/ buffer_i = 0; - while (!isspace(setting_pointers[setting_i][buffer_i])) + while (!isspace(setting_pointers[setting_i][buffer_i])) { buffer_i++; + } setting_pointers[setting_i][buffer_i] = '\0'; break; /* have read setting; don't compare with others */ - } - } + } /* end if possible name match */ + } /* end for each setting name */ } while (found_setting); done: FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__s3comms_load_aws_creds_from_file() */ -} /* H5FD__s3comms_load_aws_creds_from_file */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_load_aws_profile() @@ -2688,6 +2795,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, FILE *credfile = NULL; char awspath[117]; char filepath[128]; + int ret = 0; FUNC_ENTER_NOAPI_NOINIT @@ -2696,47 +2804,61 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, #endif /* TODO: Windows and other path gotchas */ - if (117 < HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME"))) + ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")) ; + if (ret < 0 || (size_t)ret >= 117) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format home-aws path") - if (128 < HDsnprintf(filepath, 128, "%s%s", awspath, "credentials")) + } + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "credentials"); + if (ret < 0 || (size_t)ret >= 128) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format credentials path") + } credfile = fopen(filepath, "r"); if (credfile != NULL) { - if (FAIL == H5FD__s3comms_load_aws_creds_from_file( + if (H5FD__s3comms_load_aws_creds_from_file( credfile, profile_name, key_id_out, secret_access_key_out, - aws_region_out)) + aws_region_out) + == FAIL) + { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws credentials") - if (EOF == fclose(credfile)) + } + if (fclose(credfile) == EOF) { HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close credentials file") + } credfile = NULL; - } + } /* end if credential file opened */ - if (128 < HDsnprintf(filepath, 128, "%s%s", awspath, "config")) + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "config"); + if (ret < 0 || (size_t)ret >= 128) { HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format config path") + } credfile = fopen(filepath, "r"); if (credfile != NULL) { - if (FAIL == H5FD__s3comms_load_aws_creds_from_file( + if (H5FD__s3comms_load_aws_creds_from_file( credfile, profile_name, (*key_id_out == 0) ? key_id_out : NULL, (*secret_access_key_out == 0) ? secret_access_key_out : NULL, - (*aws_region_out == 0) ? aws_region_out : NULL)) + (*aws_region_out == 0) ? aws_region_out : NULL) + == FAIL) + { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws config") - if (EOF == fclose(credfile)) + } + if (fclose(credfile) == EOF) { HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close config file") + } credfile = NULL; - } + } /* end if credential file opened */ /* fail if not all three settings were loaded */ if (*key_id_out == 0 || @@ -2748,16 +2870,15 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, done: if (credfile != NULL) { - if (EOF == fclose(credfile)) + if (fclose(credfile) == EOF) { HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL, "problem error-closing aws configuration file") + } } - FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_load_aws_profile() */ -} /* H5FD_s3comms_load_aws_profile */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_nlowercase() @@ -2787,9 +2908,10 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_nlowercase(char *dest, - const char *s, - size_t len) +H5FD_s3comms_nlowercase( + char *dest, + const char *s, + size_t len) { herr_t ret_value = SUCCEED; @@ -2814,10 +2936,9 @@ H5FD_s3comms_nlowercase(char *dest, done: FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_nlowercase() */ -} /* H5FD_s3comms_nlowercase */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_parse_url() @@ -2852,8 +2973,9 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_parse_url(const char *str, - parsed_url_t **_purl) +H5FD_s3comms_parse_url( + const char *str, + parsed_url_t **_purl) { parsed_url_t *purl = NULL; /* pointer to new structure */ const char *tmpstr = NULL; /* working pointer in string */ @@ -2863,8 +2985,6 @@ H5FD_s3comms_parse_url(const char *str, unsigned int i = 0; herr_t ret_value = FAIL; - - FUNC_ENTER_NOAPI_NOINIT; #if S3COMMS_DEBUG @@ -2946,7 +3066,8 @@ H5FD_s3comms_parse_url(const char *str, tmpstr++; } tmpstr++; - } else { + } /* end if (IPv6) */ + else { while (0 != *tmpstr) { if (':' == *tmpstr || '/' == *tmpstr || @@ -2956,12 +3077,14 @@ H5FD_s3comms_parse_url(const char *str, } tmpstr++; } - } /* if IPv4 or IPv6 */ + } /* end else (IPv4) */ len = tmpstr - curstr; if (len == 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "HOST substring cannot be empty"); - } else if (len > urllen) { + } + else + if (len > urllen) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of HOST substring"); } @@ -2990,7 +3113,9 @@ H5FD_s3comms_parse_url(const char *str, if (len == 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "PORT element cannot be empty"); - } else if (len > urllen) { + } + else + if (len > urllen) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem with length of PORT substring"); } @@ -3005,12 +3130,12 @@ H5FD_s3comms_parse_url(const char *str, */ purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); if (purl->port == NULL) { /* cannot malloc */ - HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, - "can't allocate space for PORT"); + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for PORT"); } (void)HDstrncpy(purl->port, curstr, (size_t)len); purl->port[len] = 0; - } /* if PORT element */ + } /* end if PORT element */ /************* * READ PATH * @@ -3040,7 +3165,7 @@ H5FD_s3comms_parse_url(const char *str, (void)HDstrncpy(purl->path, curstr, (size_t)len); purl->path[len] = 0; } - } /* if PATH element */ + } /* end if PATH element */ /************** * READ QUERY * @@ -3067,9 +3192,7 @@ H5FD_s3comms_parse_url(const char *str, } /* cannot malloc path pointer */ (void)HDstrncpy(purl->query, curstr, (size_t)len); purl->query[len] = 0; - } /* if QUERY exists */ - - + } /* end if QUERY exists */ *_purl = purl; ret_value = SUCCEED; @@ -3079,10 +3202,9 @@ done: H5FD_s3comms_free_purl(purl); } FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_parse_url() */ -} /* H5FD_s3comms_parse_url */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_percent_encode_char() @@ -3129,15 +3251,12 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_percent_encode_char(char *repr, - const unsigned char c, - size_t *repr_len) +H5FD_s3comms_percent_encode_char( + char *repr, + const unsigned char c, + size_t *repr_len) { - unsigned int acc = 0; unsigned int i = 0; - unsigned int k = 0; - unsigned int stack[4] = {0, 0, 0, 0}; - unsigned int stack_size = 0; int chars_written = 0; herr_t ret_value = SUCCEED; #if S3COMMS_DEBUG @@ -3145,8 +3264,6 @@ H5FD_s3comms_percent_encode_char(char *repr, unsigned char hex[3] = {0, 0, 0}; #endif - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG @@ -3172,13 +3289,19 @@ H5FD_s3comms_percent_encode_char(char *repr, #endif *repr_len = 3; chars_written = HDsnprintf(repr, 4, "%%%02X", c); - if (chars_written != 3) + if (chars_written < 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot write char %c", c); - } else { + } + } /* end if single-byte unicode char */ + else { /* multi-byte, multi-percent representation */ + unsigned int acc = 0; /* byte accumulator */ + unsigned int k = 0; /* uint character representation */ + unsigned int stack_size = 0; + unsigned char stack[4] = {0, 0, 0, 0}; #if S3COMMS_DEBUG HDfprintf(stdout, " MULTI-BYTE\n"); #endif @@ -3191,21 +3314,22 @@ H5FD_s3comms_percent_encode_char(char *repr, acc = k; acc >>= 6; /* cull least */ acc <<= 6; /* six bits */ - stack[stack_size++] = k - acc; /* max six-bit number */ + stack[stack_size++] = (unsigned char)(k - acc); k = acc >> 6; } while (k > 0); - /* now have "stack" of two to four six-bit numbers - * to be put into UTF-8 byte fields + /* `stack` now has two to four six-bit 'numbers' to be put into + * UTF-8 byte fields. */ #if S3COMMS_DEBUG HDfprintf(stdout, " STACK:\n {\n"); for (i = 0; i < stack_size; i++) { - H5FD_s3comms_bytes_to_hex((char *)hex, - (unsigned char *)(&stack[i]), - 1, - FALSE); + H5FD_s3comms_bytes_to_hex( + (char *)hex, + (&stack[i]), + 1, + FALSE); hex[2] = 0; HDfprintf(stdout, " %s,\n", hex); } @@ -3218,15 +3342,20 @@ H5FD_s3comms_percent_encode_char(char *repr, /* prepend 11[1[1]]0 to first byte */ /* 110xxxxx, 1110xxxx, or 11110xxx */ - acc = 0xC0; /* 2^7 + 2^6 -> 11000000 */ - acc += (stack_size > 2) ? 0x20 : 0; - acc += (stack_size > 3) ? 0x10 : 0; - stack_size -= 1; - chars_written = HDsnprintf(repr, 4, "%%%02X", acc + stack[stack_size]); - if (chars_written != 3) + acc = 0xC0; /* 0x11000000 */ + acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */ + acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */ + stack_size--; + chars_written = HDsnprintf( + repr, + 4, + "%%%02X", + (unsigned char)(acc + stack[stack_size])); + if (chars_written < 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot write char %c", c); + } *repr_len += 3; /************************ @@ -3235,25 +3364,27 @@ H5FD_s3comms_percent_encode_char(char *repr, /* 10xxxxxx */ for (i = 0; i < stack_size; i++) { - chars_written = HDsnprintf(&repr[i*3 + 3], - 4, - "%%%02X", - 128 + stack[stack_size - 1 - i]); - if (chars_written != 3) + chars_written = HDsnprintf( + &repr[i*3 + 3], + 4, + "%%%02X", + (unsigned char)(0x80 + stack[stack_size - 1 - i])); + if (chars_written < 0) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cannot write char %c", c); + } *repr_len += 3; - } - } + } /* end for each continuation byte */ + } /* end else (multi-byte) */ + *(repr + *repr_len) = '\0'; done: FUNC_LEAVE_NOAPI(ret_value); - } /* H5FD_s3comms_percent_encode_char */ - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_signing_key() @@ -3301,13 +3432,15 @@ done: * *---------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD + herr_t -H5FD_s3comms_signing_key(unsigned char *md, - const char *secret, - const char *region, - const char *iso8601now) +H5FD_s3comms_signing_key( + unsigned char *md, + const char *secret, + const char *region, + const char *iso8601now) { -#ifdef H5_HAVE_ROS3_VFD char *AWS4_secret = NULL; size_t AWS4_secret_len = 0; unsigned char datekey[SHA256_DIGEST_LENGTH]; @@ -3315,16 +3448,9 @@ H5FD_s3comms_signing_key(unsigned char *md, unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH]; int ret = 0; /* return value of HDsnprintf */ herr_t ret_value = SUCCEED; -#else - herr_t ret_value = SUCCEED; -#endif /* H5_HAVE_ROS3_VFD */ - - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD - #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_signing_key.\n"); #endif @@ -3356,10 +3482,11 @@ H5FD_s3comms_signing_key(unsigned char *md, /* prepend "AWS4" to start of the secret key */ ret = HDsnprintf(AWS4_secret, AWS4_secret_len,"%s%s", "AWS4", secret); - if ((size_t)ret != (AWS4_secret_len - 1)) + if ((size_t)ret != (AWS4_secret_len - 1)) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem writing AWS4+secret `%s`", secret); + } /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest * we know digest length, so ignore via NULL @@ -3396,13 +3523,26 @@ H5FD_s3comms_signing_key(unsigned char *md, done: H5MM_xfree(AWS4_secret); -#endif /* H5_HAVE_ROS3_VFD */ - FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_signing_key() */ + +#else /* H5_HAVE_ROS3_VFD not defined */ -} /* H5FD_s3comms_signing_key */ +herr_t +H5FD_s3comms_signing_key( + unsigned char H5_ATTR_UNUSED *md, + const char H5_ATTR_UNUSED *secret, + const char H5_ATTR_UNUSED *region, + const char H5_ATTR_UNUSED *iso8601now) +{ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI_NOINIT_NOERR; + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_signing_key() */ +#endif /* H5_HAVE_ROS3_VFD */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_tostringtosign() @@ -3450,13 +3590,14 @@ done: * *---------------------------------------------------------------------------- */ +#ifdef H5_HAVE_ROS3_VFD herr_t -H5FD_s3comms_tostringtosign(char *dest, - const char *req, - const char *now, - const char *region) +H5FD_s3comms_tostringtosign( + char *dest, + const char *req, + const char *now, + const char *region) { -#ifdef H5_HAVE_ROS3_VFD unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1]; size_t d = 0; char day[9]; @@ -3465,15 +3606,9 @@ H5FD_s3comms_tostringtosign(char *dest, int ret = 0; /* HDsnprintf return value */ herr_t ret_value = SUCCEED; char tmp[128]; -#else - herr_t ret_value = FAIL; -#endif /* H5_HAVE_ROS3_VFD */ - FUNC_ENTER_NOAPI_NOINIT -#ifdef H5_HAVE_ROS3_VFD - #if S3COMMS_DEBUG HDfprintf(stdout, "called H5FD_s3comms_tostringtosign.\n"); #endif @@ -3495,8 +3630,6 @@ H5FD_s3comms_tostringtosign(char *dest, "Region cannot be NULL.\n") } - - for (i = 0; i < 128; i++) { tmp[i] = '\0'; } @@ -3507,11 +3640,10 @@ H5FD_s3comms_tostringtosign(char *dest, HDstrncpy(day, now, 8); day[8] = '\0'; ret = HDsnprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region); - if (ret <= 0 || ret >= 127) + if (ret <= 0 || ret >= 127) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem adding day and region to string") - - + } HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); d = 17; @@ -3528,11 +3660,12 @@ H5FD_s3comms_tostringtosign(char *dest, HDstrlen(req), checksum); - if (FAIL == - H5FD_s3comms_bytes_to_hex(hexsum, - (const unsigned char *)checksum, - SHA256_DIGEST_LENGTH, - true)) + if (H5FD_s3comms_bytes_to_hex( + hexsum, + (const unsigned char *)checksum, + SHA256_DIGEST_LENGTH, + true) + == FAIL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "could not create hex string"); @@ -3544,14 +3677,27 @@ H5FD_s3comms_tostringtosign(char *dest, dest[d] = '\0'; -#endif /* H5_HAVE_ROS3_VFD */ - done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ros3_tostringtosign() */ + +#else /* H5_HAVE_ROS3_VFD not defined */ -} /* H5ros3_tostringtosign */ +herr_t +H5FD_s3comms_tostringtosign( + char H5_ATTR_UNUSED *dest, + const char H5_ATTR_UNUSED *req, + const char H5_ATTR_UNUSED *now, + const char H5_ATTR_UNUSED *region) +{ + herr_t ret_value = FAIL; + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ros3_tostringtosign() */ +#endif /* H5_HAVE_ROS3_VFD */ + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_trim() @@ -3594,9 +3740,7 @@ H5FD_s3comms_trim(char *dest, size_t s_len, size_t *n_written) { - herr_t ret_value = SUCCEED; - - + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -3612,8 +3756,6 @@ H5FD_s3comms_trim(char *dest, s_len = 0; } - - if (s_len > 0) { /* Find first non-whitespace character from start; * reduce total length per character. @@ -3645,10 +3787,9 @@ H5FD_s3comms_trim(char *dest, done: FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_trim() */ -} /* H5FD_s3comms_trim */ - - + /*---------------------------------------------------------------------------- * * Function: H5FD_s3comms_uriencode() @@ -3698,11 +3839,12 @@ done: *---------------------------------------------------------------------------- */ herr_t -H5FD_s3comms_uriencode(char *dest, - const char *s, - size_t s_len, - hbool_t encode_slash, - size_t *n_written) +H5FD_s3comms_uriencode( + char *dest, + const char *s, + size_t s_len, + hbool_t encode_slash, + size_t *n_written) { char c = 0; size_t dest_off = 0; @@ -3712,20 +3854,20 @@ H5FD_s3comms_uriencode(char *dest, herr_t ret_value = SUCCEED; size_t s_off = 0; - - FUNC_ENTER_NOAPI_NOINIT #if S3COMMS_DEBUG HDfprintf(stdout, "H5FD_s3comms_uriencode called.\n"); #endif - if (s == NULL) + if (s == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "source string cannot be NULL"); - if (dest == NULL) + } + if (dest == NULL) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "destination cannot be NULL"); + } /* Write characters to destination, converting to percent-encoded * "hex-utf-8" strings if necessary. @@ -3741,12 +3883,14 @@ H5FD_s3comms_uriencode(char *dest, (c == '/' && encode_slash == FALSE)) { dest[dest_off++] = c; - } else { + } + else { hex_off = 0; - if (FAIL == - H5FD_s3comms_percent_encode_char(hex_buffer, - (const unsigned char)c, - &hex_len)) + if (H5FD_s3comms_percent_encode_char( + hex_buffer, + (const unsigned char)c, + &hex_len) + == FAIL) { hex_buffer[0] = c; hex_buffer[1] = 0; @@ -3758,8 +3902,8 @@ H5FD_s3comms_uriencode(char *dest, for (hex_off = 0; hex_off < hex_len; hex_off++) { dest[dest_off++] = hex_buffer[hex_off]; } - } - } + } /* end else (not a regular character) */ + } /* end for each character */ if (dest_off < s_len) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, @@ -3769,7 +3913,6 @@ H5FD_s3comms_uriencode(char *dest, done: FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_s3comms_uriencode */ diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h index 93a5a81..16f6a8b 100644 --- a/src/H5FDs3comms.h +++ b/src/H5FDs3comms.h @@ -577,7 +577,9 @@ H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, H5_DLL struct tm * gmnow(void); herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, + int cr_size, char *signed_headers_dest, + int sh_size, hrb_t *http_request); H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest, |