diff options
Diffstat (limited to 'src/H5FDhdfs.c')
| -rw-r--r-- | src/H5FDhdfs.c | 1782 |
1 files changed, 843 insertions, 939 deletions
diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c index 0f18318..02981d5 100644 --- a/src/H5FDhdfs.c +++ b/src/H5FDhdfs.c @@ -34,8 +34,20 @@ #include "H5Pprivate.h" /* Property lists */ #ifdef H5_HAVE_LIBHDFS -#include "hdfs.h" -#endif + +/* HDFS routines + * Have to turn off -Wstrict-prototypes as this header contains functions + * defined as foo() instead of foo(void), which triggers warnings that HDF5 + * then interprets as errors. + * -Wundef isn't interpreted as an error by HDF5, but the header does do + * some bad symbol interpretation that raises a warning that is out of our + * control. + */ +H5_GCC_DIAG_OFF(strict - prototypes) +H5_GCC_DIAG_OFF(undef) +#include <hdfs.h> +H5_GCC_DIAG_ON(strict - prototypes) +H5_GCC_DIAG_ON(undef) /* toggle function call prints: 1 turns on */ #define HDFS_DEBUG 0 @@ -117,8 +129,6 @@ static unsigned long long hdfs_stats_boundaries[HDFS_STATS_BIN_COUNT]; * * Programmer: Jacob Smith * - * Changes: None - * ***************************************************************************/ typedef struct { unsigned long long count; @@ -129,8 +139,6 @@ typedef struct { #endif /* HDFS_STATS */ -#ifdef H5_HAVE_LIBHDFS - /* "unique" identifier for `hdfs_t` structures. * Randomly generated by unweighted dice rolls. */ @@ -144,9 +152,7 @@ typedef struct { * * Contain/retain information associated with a file hosted on Hadoop * Distributed File System (HDFS). Instantiated and populated via - * `H5FD_hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`. - * - * + * `H5FD__hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`. * * `magic` (unisgned long) * @@ -173,8 +179,6 @@ typedef struct { * Programmer: Jacob Smith * May 2018 * - * Changes: None - * *************************************************************************** */ typedef struct { @@ -184,168 +188,6 @@ typedef struct { hdfsFile file; } hdfs_t; -/*-------------------------------------------------------------------------- - * Function: H5FD_hdfs_handle_open - * - * Purpose: Create a HDFS file handle, 'opening' the target file. - * - * Return: Success: Pointer to HDFS container/handle of opened file. - * Failure: NULL - * - * Programmer: Gerd Herber - * May 2018 - * - * Changes: None. - *-------------------------------------------------------------------------- - */ -static hdfs_t * -H5FD_hdfs_handle_open(const char *path, const char *namenode_name, const int32_t namenode_port, - const char *user_name, const char *kerberos_ticket_cache, - const int32_t stream_buffer_size) -{ - struct hdfsBuilder *builder = NULL; - hdfs_t * handle = NULL; - hdfs_t * ret_value = NULL; - - FUNC_ENTER_NOAPI_NOINIT - -#if HDFS_DEBUG - HDfprintf(stdout, "called H5FD_hdfs_handle_open.\n"); -#endif - - if (path == NULL || path[0] == '\0') { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "path cannot be null.\n") - } - if (namenode_name == NULL /* || namenode_name[0] == '\0' */) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "namenode name cannot be null.\n") - } - if (namenode_port < 0 || namenode_port > 65535) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "namenode port must be non-negative and <= 65535.\n") - } - if (stream_buffer_size < 0) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "buffer size must non-negative.\n") - } - - handle = (hdfs_t *)H5MM_malloc(sizeof(hdfs_t)); - if (handle == NULL) { - HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "could not malloc space for handle.\n") - } - - handle->magic = (unsigned long)HDFS_HDFST_MAGIC; - handle->filesystem = NULL; /* TODO: not a pointer; NULL may cause bug */ - handle->fileinfo = NULL; - handle->file = NULL; /* TODO: not a pointer; NULL may cause bug */ - - builder = hdfsNewBuilder(); - if (!builder) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "(hdfs) failed to create builder") - } - hdfsBuilderSetNameNode(builder, namenode_name); - hdfsBuilderSetNameNodePort(builder, (tPort)namenode_port); - if (user_name != NULL && user_name[0] != '\0') { - hdfsBuilderSetUserName(builder, user_name); - } - if (kerberos_ticket_cache != NULL && kerberos_ticket_cache[0] != '\0') { - hdfsBuilderSetKerbTicketCachePath(builder, kerberos_ticket_cache); - } - /* Call to `hdfsBuilderConnect` releases builder, regardless of success. */ - handle->filesystem = hdfsBuilderConnect(builder); - if (!handle->filesystem) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "(hdfs) could not connect to default namenode") - } - handle->fileinfo = hdfsGetPathInfo(handle->filesystem, path); - if (!handle->fileinfo) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "hdfsGetPathInfo failed") - } - handle->file = hdfsOpenFile(handle->filesystem, path, O_RDONLY, stream_buffer_size, 0, 0); - if (!handle->file) { - HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "(hdfs) could not open") - } - - ret_value = handle; - -done: - if (ret_value == NULL && handle != NULL) { - /* error; clean up */ - HDassert(handle->magic == HDFS_HDFST_MAGIC); - handle->magic++; - if (handle->file != NULL) { - if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close hdfs file handle") - } - } - if (handle->fileinfo != NULL) { - hdfsFreeFileInfo(handle->fileinfo, 1); - } - if (handle->filesystem != NULL) { - if (FAIL == (hdfsDisconnect(handle->filesystem))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to disconnect from hdfs") - } - } - H5MM_xfree(handle); - } - - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD_hdfs_handle_open() */ - -/*-------------------------------------------------------------------------- - * Function: H5FD_hdfs_handle_close - * - * Purpose: 'Close' an HDFS file container/handle, releasing underlying - * resources. - * - * Return: Success: `SUCCEED` (0) - * Failure: `FAIL` (-1) - * - * Programmer: Gerd Herber - * May 2018 - * - * Changes: None. - *-------------------------------------------------------------------------- - */ -static herr_t -H5FD_hdfs_handle_close(hdfs_t *handle) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - -#if HDFS_DEBUG - HDfprintf(stdout, "called H5FD_hdfs_close.\n"); -#endif - - if (handle == NULL) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle cannot be null.\n") - } - if (handle->magic != HDFS_HDFST_MAGIC) { - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle has invalid magic.\n") - } - - handle->magic++; - if (handle->file != NULL) { - if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close hdfs file handle") - } - } - if (handle->fileinfo != NULL) { - hdfsFreeFileInfo(handle->fileinfo, 1); - } - if (handle->filesystem != NULL) { - if (FAIL == (hdfsDisconnect(handle->filesystem))) { - HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to disconnect hdfs file system") - } - } - - H5MM_xfree(handle); - -done: - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5FD_hdfs_close() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*************************************************************************** * * Structure: H5FD_hdfs_t @@ -402,17 +244,13 @@ done: * * Programmer: Jacob Smith * - * Changes: None. - * *************************************************************************** */ typedef struct H5FD_hdfs_t { H5FD_t pub; H5FD_hdfs_fapl_t fa; haddr_t eoa; -#ifdef H5_HAVE_LIBHDFS - hdfs_t *hdfs_handle; -#endif + hdfs_t * hdfs_handle; #if HDFS_STATS hdfs_statsbin meta[HDFS_STATS_BIN_COUNT + 1]; hdfs_statsbin raw[HDFS_STATS_BIN_COUNT + 1]; @@ -429,10 +267,8 @@ typedef struct H5FD_hdfs_t { * Only included if HDFS code should compile. * */ -#define MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1) -#ifdef H5_HAVE_LIBHDFS +#define MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1) #define ADDR_OVERFLOW(A) (HADDR_UNDEF == (A) || ((A) & ~(haddr_t)MAXADDR)) -#endif /* H5_HAVE_LIBHDFS */ /* Prototypes */ static void * H5FD_hdfs_fapl_get(H5FD_t *_file); @@ -489,10 +325,8 @@ static const H5FD_class_t H5FD_hdfs_g = { H5FD_FLMAP_DICHOTOMY /* fl_map */ }; -#ifdef H5_HAVE_LIBHDFS /* Declare a free list to manage the H5FD_hdfs_t struct */ H5FL_DEFINE_STATIC(H5FD_hdfs_t); -#endif /* H5_HAVE_LIBHDFS */ /*------------------------------------------------------------------------- * Function: H5FD_hdfs_init_interface @@ -521,21 +355,22 @@ H5FD_hdfs_init_interface(void) * Return: Success: The driver ID for the hdfs driver. * Failure: Negative * - * Programmer: Robb Matzke - * Thursday, July 29, 1999 + * Programmer: Jacob Smith, 2018 * *------------------------------------------------------------------------- */ hid_t H5FD_hdfs_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ +#if HDFS_STATS unsigned int bin_i; +#endif + hid_t ret_value = H5I_INVALID_HID; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI(H5I_INVALID_HID) #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_init() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif if (H5I_VFL != H5I_get_type(H5FD_HDFS_g)) @@ -546,6 +381,7 @@ H5FD_hdfs_init(void) */ for (bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) { unsigned long long value = 0; + HDFS_STATS_POW(bin_i, &value) hdfs_stats_boundaries[bin_i] = value; } @@ -558,7 +394,7 @@ done: } /* end H5FD_hdfs_init() */ /*--------------------------------------------------------------------------- - * Function: H5FD_log_term + * Function: H5FD_hdfs_term * * Purpose: Shut down the VFD * @@ -574,7 +410,7 @@ H5FD_hdfs_term(void) FUNC_ENTER_NOAPI_NOINIT_NOERR #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_term() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif /* Reset VFL ID */ @@ -583,6 +419,180 @@ H5FD_hdfs_term(void) FUNC_LEAVE_NOAPI_VOID } /* end H5FD_hdfs_term() */ +/*-------------------------------------------------------------------------- + * Function: H5FD_hdfs_handle_open + * + * Purpose: Create a HDFS file handle, 'opening' the target file. + * + * Return: Success: Pointer to HDFS container/handle of opened file. + * Failure: NULL + * + * Programmer: Gerd Herber + * May 2018 + * + *-------------------------------------------------------------------------- + */ +static hdfs_t * +H5FD_hdfs_handle_open(const char *path, const char *namenode_name, const int32_t namenode_port, + const char *user_name, const char *kerberos_ticket_cache, + const int32_t stream_buffer_size) +{ + struct hdfsBuilder *builder = NULL; + hdfs_t * handle = NULL; + hdfs_t * ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "called %s.\n", FUNC); +#endif + + if (path == NULL || path[0] == '\0') + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "path cannot be null") + if (namenode_name == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "namenode name cannot be null") + if (namenode_port < 0 || namenode_port > 65535) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "namenode port must be non-negative and <= 65535") + if (stream_buffer_size < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "buffer size must non-negative") + + handle = (hdfs_t *)H5MM_malloc(sizeof(hdfs_t)); + if (handle == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "could not malloc space for handle") + + handle->magic = (unsigned long)HDFS_HDFST_MAGIC; + handle->filesystem = NULL; /* TODO: not a pointer; NULL may cause bug */ + handle->fileinfo = NULL; + handle->file = NULL; /* TODO: not a pointer; NULL may cause bug */ + + builder = hdfsNewBuilder(); + if (!builder) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "(hdfs) failed to create builder") + hdfsBuilderSetNameNode(builder, namenode_name); + hdfsBuilderSetNameNodePort(builder, (tPort)namenode_port); + if (user_name != NULL && user_name[0] != '\0') + hdfsBuilderSetUserName(builder, user_name); + if (kerberos_ticket_cache != NULL && kerberos_ticket_cache[0] != '\0') + hdfsBuilderSetKerbTicketCachePath(builder, kerberos_ticket_cache); + + /* Call to `hdfsBuilderConnect` releases builder, regardless of success. */ + handle->filesystem = hdfsBuilderConnect(builder); + if (!handle->filesystem) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "(hdfs) could not connect to default namenode") + handle->fileinfo = hdfsGetPathInfo(handle->filesystem, path); + if (!handle->fileinfo) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "hdfsGetPathInfo failed") + handle->file = hdfsOpenFile(handle->filesystem, path, O_RDONLY, stream_buffer_size, 0, 0); + if (!handle->file) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "(hdfs) could not open") + + ret_value = handle; + +done: + if (ret_value == NULL && handle != NULL) { + /* error; clean up */ + HDassert(handle->magic == HDFS_HDFST_MAGIC); + handle->magic++; + if (handle->file != NULL) + if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close hdfs file handle") + if (handle->fileinfo != NULL) + hdfsFreeFileInfo(handle->fileinfo, 1); + if (handle->filesystem != NULL) + if (FAIL == (hdfsDisconnect(handle->filesystem))) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to disconnect from hdfs") + H5MM_xfree(handle); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__hdfs_handle_open() */ + +/*-------------------------------------------------------------------------- + * Function: H5FD_hdfs_handle_close + * + * Purpose: 'Close' an HDFS file container/handle, releasing underlying + * resources. + * + * Return: Success: `SUCCEED` (0) + * Failure: `FAIL` (-1) + * + * Programmer: Gerd Herber + * May 2018 + * + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_handle_close(hdfs_t *handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "called %s.\n", FUNC); +#endif + + if (handle == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle cannot be null") + if (handle->magic != HDFS_HDFST_MAGIC) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "handle has invalid magic") + + handle->magic++; + if (handle->file != NULL) + if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close hdfs file handle") + if (handle->fileinfo != NULL) + hdfsFreeFileInfo(handle->fileinfo, 1); + if (handle->filesystem != NULL) + if (FAIL == (hdfsDisconnect(handle->filesystem))) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to disconnect hdfs file system") + + H5MM_xfree(handle); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__hdfs_handle_close() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_validate_config() + * + * Purpose: Test to see if the supplied instance of H5FD_hdfs_fapl_t + * contains internally consistant data. Return SUCCEED if so, + * and FAIL otherwise. + * + * Note the difference between internally consistant and + * correct. As we will have to try to access the target + * object to determine whether the supplied data is correct, + * we will settle for internal consistancy at this point + * + * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally + * consistant data, FAIL otherwise. + * + * Programmer: Jacob Smith + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t *fa) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(fa != NULL); + + if (fa->version != H5FD__CURR_HDFS_FAPL_T_VERSION) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unknown H5FD_hdfs_fapl_t version"); + if (fa->namenode_port > 65535) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number"); + if (fa->namenode_port < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_validate_config() */ + /*------------------------------------------------------------------------- * Function: H5Pset_fapl_hdfs * @@ -609,12 +619,12 @@ H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa) HDassert(fa != NULL); #if HDFS_DEBUG - HDfprintf(stdout, "H5Pset_fapl_hdfs() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif - if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - if (FAIL == H5FD_hdfs_validate_config(fa)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid hdfs config") @@ -625,35 +635,246 @@ done: } /* H5Pset_fapl_hdfs() */ /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_open() + * Function: H5Pget_fapl_hdfs + * + * Purpose: Returns information about the hdfs file access property + * list though the function arguments. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: John Mainzer + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out) +{ + const H5FD_hdfs_fapl_t *fa = NULL; + H5P_genplist_t * plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + +#if HDFS_DEBUG + HDfprintf(stdout, "called %s.\n", FUNC); +#endif + + if (fa_out == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL") + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + if (H5FD_HDFS != H5P_get_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + + fa = (const H5FD_hdfs_fapl_t *)H5P_get_driver_info(plist); + if (fa == NULL) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") + + /* Copy the hdfs fapl data out */ + HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_fapl_hdfs() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_get + * + * Purpose: Gets a file access property list which could be used to + * create an identical file. + * + * Return: Success: Ptr to new file access property list value. + * + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_hdfs_fapl_get(H5FD_t *_file) +{ + H5FD_hdfs_t * file = (H5FD_hdfs_t *)_file; + H5FD_hdfs_fapl_t *fa = NULL; + void * ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t)); + if (fa == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed") + + /* Copy the fields of the structure */ + HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); + + ret_value = fa; + +done: + if (ret_value == NULL && fa != NULL) + H5MM_xfree(fa); /* clean up on error */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_fapl_get() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_copy + * + * Purpose: Copies the hdfs-specific file access properties. * - * Purpose: Create and/or opens a file as an HDF5 file. + * Return: Success: Ptr to a new property list * - * Any flag except H5F_ACC_RDONLY will cause an error. + * Failure: NULL * - * Return: Success: A pointer to a new file data structure. - * The public fields will be initialized by the caller, which is - * always H5FD_open(). - * Failure: NULL + * Programmer: John Mainzer + * 9/8/17 + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_hdfs_fapl_copy(const void *_old_fa) +{ + const H5FD_hdfs_fapl_t *old_fa = (const H5FD_hdfs_fapl_t *)_old_fa; + H5FD_hdfs_fapl_t * new_fa = NULL; + void * ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t)); + if (new_fa == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed") + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); + ret_value = new_fa; + +done: + if (ret_value == NULL && new_fa != NULL) + H5MM_xfree(new_fa); /* clean up on error */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_fapl_copy() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_free + * + * Purpose: Frees the hdfs-specific file access properties. + * + * Return: SUCCEED (cannot fail) + * + * Programmer: John Mainzer + * 9/8/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_fapl_free(void *_fa) +{ + H5FD_hdfs_fapl_t *fa = (H5FD_hdfs_fapl_t *)_fa; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(fa != NULL); /* sanity check */ + + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5FD_hdfs_fapl_free() */ + +#if HDFS_STATS +/*---------------------------------------------------------------------------- + * + * Function: hdfs_reset_stats() + * + * Purpose: + * + * Reset the stats collection elements in this virtual file structure. + * + * Clears any set data in stats bins; initializes/zeroes values. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - Occurs if the file is invalid somehow + * + * Programmer: Jacob Smith + * 2017-12-08 + * + *---------------------------------------------------------------------------- + */ +static herr_t +hdfs_reset_stats(H5FD_hdfs_t *file) +{ + unsigned i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDprintf("hdfs_reset_stats() called\n"); +#endif + + if (file == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null") + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + file->raw[i].bytes = 0; + file->raw[i].count = 0; + file->raw[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; + file->raw[i].max = 0; + + file->meta[i].bytes = 0; + file->meta[i].count = 0; + file->meta[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; + file->meta[i].max = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* hdfs_reset_stats */ +#endif /* HDFS_STATS */ + +/*------------------------------------------------------------------------- + * + * Function: H5FD_hdfs_open() + * + * Purpose: + * + * Create and/or opens a file as an HDF5 file. + * + * Any flag except H5F_ACC_RDONLY will cause an error. + * + * Return: + * + * Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is + * always H5FD_open(). + * + * Failure: NULL * * Programmer: Jacob Smith * 2017-11-02 * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS static H5FD_t * H5FD_hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { - H5FD_hdfs_t * file = NULL; - hdfs_t * handle = NULL; - H5FD_hdfs_fapl_t fa; H5FD_t * ret_value = NULL; + H5FD_hdfs_t * file = NULL; + hdfs_t * handle = NULL; + H5FD_hdfs_fapl_t fa; FUNC_ENTER_NOAPI_NOINIT #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_open() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif /* HDFS_DEBUG */ /* Sanity check on file offsets */ @@ -666,7 +887,6 @@ H5FD_hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") if (ADDR_OVERFLOW(maxaddr)) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") - if (flags != H5F_ACC_RDONLY) HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed") if (fapl_id == H5P_DEFAULT || fapl_id == H5P_FILE_ACCESS_DEFAULT) @@ -676,16 +896,15 @@ H5FD_hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr) handle = H5FD_hdfs_handle_open(path, fa.namenode_name, fa.namenode_port, fa.user_name, fa.kerberos_ticket_cache, fa.stream_buffer_size); - if (handle == NULL) HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open") HDassert(handle->magic == HDFS_HDFST_MAGIC); - /* Create the new file struct */ - if (NULL == (file = H5FL_CALLOC(H5FD_hdfs_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") - + /* Create new file struct */ + file = H5FL_CALLOC(H5FD_hdfs_t); + if (file == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct") file->hdfs_handle = handle; HDmemcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t)); @@ -697,62 +916,304 @@ H5FD_hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr) ret_value = (H5FD_t *)file; done: - if (NULL == ret_value) { - if (handle != NULL) { + if (ret_value == NULL) { + if (handle != NULL) if (FAIL == H5FD_hdfs_handle_close(handle)) HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close HDFS file handle") - } - if (file != NULL) { + if (file != NULL) file = H5FL_FREE(H5FD_hdfs_t, file); - } } /* end if null return value (error) */ FUNC_LEAVE_NOAPI(ret_value) - } /* H5FD_hdfs_open() */ -#else /* H5_HAVE_LIBHDFS not defined */ +#if HDFS_STATS -static H5FD_t * -H5FD_hdfs_open(const char H5_ATTR_UNUSED *path, unsigned H5_ATTR_UNUSED flags, hid_t H5_ATTR_UNUSED fapl_id, - haddr_t H5_ATTR_UNUSED maxaddr) +/*---------------------------------------------------------------------------- + * + * Function: hdfs_fprint_stats() + * + * Purpose: + * + * Tabulate and pretty-print statistics for this virtual file. + * + * Should be called upon file close. + * + * Shows number of reads and bytes read, broken down by + * "raw" (H5FD_MEM_DRAW) + * or "meta" (any other flag) + * + * Prints filename and listing of total number of reads and bytes read, + * both as a grand total and separate meta- and rawdata reads. + * + * If any reads were done, prints out two tables: + * + * 1. overview of raw- and metadata reads + * - min (smallest size read) + * - average of size read + * - k,M,G suffixes by powers of 1024 (2^10) + * - max (largest size read) + * 2. tabulation of "bins", sepraring reads into exponentially-larger + * ranges of size. + * - columns for number of reads, total bytes, and average size, with + * separate sub-colums for raw- and metadata reads. + * - each row represents one bin, identified by the top of its range + * + * Bin ranges can be modified with pound-defines at the top of this file. + * + * Bins without any reads in their bounds are not printed. + * + * An "overflow" bin is also present, to catch "big" reads. + * + * Output for all bins (and range ceiling and average size report) + * is divied by powers of 1024. By corollary, four digits before the decimal + * is valid. + * + * - 41080 bytes is represented by 40.177k, not 41.080k + * - 1004.831M represents approx. 1052642000 bytes + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - occurs if the file passed in is invalid + * - TODO: if stream is invalid? how can we check this? + * + * Programmer: Jacob Smith + * + *---------------------------------------------------------------------------- + */ +static herr_t +hdfs_fprint_stats(FILE *stream, const H5FD_hdfs_t *file) { - H5FD_t *ret_value = NULL; + herr_t ret_value = SUCCEED; + parsed_url_t * purl = NULL; + unsigned i = 0; + unsigned long count_meta = 0; + unsigned long count_raw = 0; + double average_meta = 0.0; + double average_raw = 0.0; + unsigned long long min_meta = (unsigned long long)HDFS_STATS_STARTING_MIN; + unsigned long long min_raw = (unsigned long long)HDFS_STATS_STARTING_MIN; + unsigned long long max_meta = 0; + unsigned long long max_raw = 0; + unsigned long long bytes_raw = 0; + unsigned long long bytes_meta = 0; + double re_dub = 0.0; /* re-usable double variable */ + unsigned suffix_i = 0; + const char suffixes[] = {' ', 'K', 'M', 'G', 'T', 'P'}; FUNC_ENTER_NOAPI_NOINIT - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "Illegal open of unsupported virtual file (hdfs)"); + if (stream == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file stream cannot be null") + if (file == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file cannot be null") + if (file->hdfs_handle == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle cannot be null") + if (file->hdfs_handle->magic != HDFS_HDFST_MAGIC) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle has invalid magic") + + /******************* + * AGGREGATE STATS * + *******************/ + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + const hdfs_statsbin *r = &file->raw[i]; + const hdfs_statsbin *m = &file->meta[i]; + + if (m->min < min_meta) + min_meta = m->min; + if (r->min < min_raw) + min_raw = r->min; + if (m->max > max_meta) + max_meta = m->max; + if (r->max > max_raw) + max_raw = r->max; + + count_raw += r->count; + count_meta += m->count; + bytes_raw += r->bytes; + bytes_meta += m->bytes; + } + if (count_raw > 0) + average_raw = (double)bytes_raw / (double)count_raw; + if (count_meta > 0) + average_meta = (double)bytes_meta / (double)count_meta; + + /****************** + * PRINT OVERVIEW * + ******************/ + + HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n", count_raw + count_meta, count_meta, + count_raw); + HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", bytes_raw + bytes_meta, bytes_meta, + bytes_raw); + + if (count_raw + count_meta == 0) + goto done; + + /************************* + * PRINT AGGREGATE STATS * + *************************/ + + HDfprintf(stream, "SIZES meta raw\n"); + HDfprintf(stream, " min "); + if (count_meta == 0) + HDfprintf(stream, " 0.000 "); + else { + re_dub = (double)min_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + } + + if (count_raw == 0) + HDfprintf(stream, " 0.000 \n"); + else { + re_dub = (double)min_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + } + + HDfprintf(stream, " avg "); + re_dub = (double)average_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)average_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + HDfprintf(stream, " max "); + re_dub = (double)max_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)max_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + /****************************** + * PRINT INDIVIDUAL BIN STATS * + ******************************/ + + HDfprintf(stream, "BINS # of reads total bytes average size\n"); + HDfprintf(stream, " up-to meta raw meta raw meta raw\n"); + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + const hdfs_statsbin *m; + const hdfs_statsbin *r; + unsigned long long range_end = 0; + char bm_suffix = ' '; /* bytes-meta */ + double bm_val = 0.0; + char br_suffix = ' '; /* bytes-raw */ + double br_val = 0.0; + char am_suffix = ' '; /* average-meta */ + double am_val = 0.0; + char ar_suffix = ' '; /* average-raw */ + double ar_val = 0.0; + + m = &file->meta[i]; + r = &file->raw[i]; + if (r->count == 0 && m->count == 0) + continue; + + range_end = hdfs_stats_boundaries[i]; + + if (i == HDFS_STATS_BIN_COUNT) { + range_end = hdfs_stats_boundaries[i - 1]; + HDfprintf(stream, ">"); + } + else + HDfprintf(stream, " "); + + bm_val = (double)m->bytes; + for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) + bm_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + bm_suffix = suffixes[suffix_i]; + + br_val = (double)r->bytes; + for (suffix_i = 0; br_val >= 1024.0; suffix_i++) + br_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + br_suffix = suffixes[suffix_i]; + + if (m->count > 0) + am_val = (double)(m->bytes) / (double)(m->count); + for (suffix_i = 0; am_val >= 1024.0; suffix_i++) + am_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + am_suffix = suffixes[suffix_i]; + + if (r->count > 0) + ar_val = (double)(r->bytes) / (double)(r->count); + for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) + ar_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + ar_suffix = suffixes[suffix_i]; + + re_dub = (double)range_end; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + + HDfprintf(stream, " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, + suffixes[suffix_i], /* bin ceiling */ + m->count, /* metadata reads */ + r->count, /* rawdata reads */ + bm_val, bm_suffix, /* metadata bytes */ + br_val, br_suffix, /* rawdata bytes */ + am_val, am_suffix, /* metadata average */ + ar_val, ar_suffix); /* rawdata average */ + HDfflush(stream); + } done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_hdfs_open() */ -#endif /* H5_HAVE_LIBHDFS */ + FUNC_LEAVE_NOAPI(ret_value); +} /* hdfs_fprint_stats */ +#endif /* HDFS_STATS */ /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_close * - * Purpose: Closes an HDF5 file. + * Function: H5FD_hdfs_close + * + * Purpose: * - * Return: Success: SUCCEED - * Failure: FAIL, file not closed. + * Close an HDF5 file. + * + * Return: + * + * Success: SUCCEED + * Failure: FAIL, file not closed. * * Programmer: Jacob Smith * 2017-11-02 * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS - static herr_t H5FD_hdfs_close(H5FD_t *_file) { - H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; herr_t ret_value = SUCCEED; + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; FUNC_ENTER_NOAPI_NOINIT #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_close() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif /* Sanity checks */ @@ -762,9 +1223,8 @@ H5FD_hdfs_close(H5FD_t *_file) /* Close the underlying request handle */ if (file->hdfs_handle != NULL) - if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) { + if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close HDFS file handle") - } #if HDFS_STATS /* TODO: mechanism to re-target stats printout */ @@ -779,52 +1239,37 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_close() */ -#else /* H5_HAVE_LIBHDFS not defined */ - -static herr_t -H5FD_hdfs_close(H5FD_t H5_ATTR_UNUSED *_file) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "Illegal close of unsupported Virtual File (hdfs)") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_hdfs_close() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_cmp() * - * Purpose: Compares two files using this driver by their HDFS-provided file info, - * field-by-field. + * Function: H5FD_hdfs_cmp() * - * Return: Equivalent: 0 - * Not Equivalent: -1 + * Purpose: + * + * Compares two files using this driver by their HDFS-provided file info, + * field-by-field. + * + * Return: + * Equivalent: 0 + * Not Equivalent: -1 * * Programmer: Gerd Herber * May 2018 * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS - static int H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2) { + int ret_value = 0; const H5FD_hdfs_t *f1 = (const H5FD_hdfs_t *)_f1; const H5FD_hdfs_t *f2 = (const H5FD_hdfs_t *)_f2; hdfsFileInfo * finfo1 = NULL; hdfsFileInfo * finfo2 = NULL; - int ret_value = 0; FUNC_ENTER_NOAPI_NOINIT_NOERR #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_cmp() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif /* HDFS_DEBUG */ HDassert(f1->hdfs_handle != NULL); @@ -855,10 +1300,10 @@ H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2) if (finfo1->mBlockSize != finfo2->mBlockSize) { HGOTO_DONE(-1); } - if (strcmp(finfo1->mOwner, finfo2->mOwner)) { + if (HDstrcmp(finfo1->mOwner, finfo2->mOwner)) { HGOTO_DONE(-1); } - if (strcmp(finfo1->mGroup, finfo2->mGroup)) { + if (HDstrcmp(finfo1->mGroup, finfo2->mGroup)) { HGOTO_DONE(-1); } if (finfo1->mPermissions != finfo2->mPermissions) { @@ -872,20 +1317,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_cmp() */ -#else /* H5_HAVE_LIBHDFS not defined */ - -static int -H5FD_hdfs_cmp(const H5FD_t H5_ATTR_UNUSED *_f1, const H5FD_t H5_ATTR_UNUSED *_f2) -{ - int ret_value = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_hdfs_cmp() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*------------------------------------------------------------------------- * Function: H5FD_hdfs_query * @@ -911,7 +1342,7 @@ H5FD_hdfs_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) /* out FUNC_ENTER_NOAPI_NOINIT_NOERR #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_query() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif if (flags) { @@ -923,21 +1354,24 @@ H5FD_hdfs_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) /* out } /* H5FD_hdfs_query() */ /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_get_eoa() * - * Purpose: Gets the end-of-address marker for the file. The EOA marker - * is the first address past the last byte allocated in the - * format address space. + * Function: H5FD_hdfs_get_eoa() + * + * Purpose: + * + * Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. * - * Return: The end-of-address marker. + * Return: + * + * The end-of-address marker. * * Programmer: Jacob Smith * 2017-11-02 * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS - static haddr_t H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { @@ -946,42 +1380,29 @@ H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) FUNC_ENTER_NOAPI_NOINIT_NOERR #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif FUNC_LEAVE_NOAPI(file->eoa) } /* end H5FD_hdfs_get_eoa() */ -#else /* H5_HAVE_LIBHDFS not defined */ - -static haddr_t -H5FD_hdfs_get_eoa(const H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type) -{ - FUNC_ENTER_NOAPI_NOINIT_NOERR - -#if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); -#endif - - FUNC_LEAVE_NOAPI(0) -} /* end H5FD_hdfs_get_eoa() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_set_eoa() * - * Purpose: Set the end-of-address marker for the file. + * Function: H5FD_hdfs_set_eoa() * - * Return: SUCCEED (Can't fail) + * Purpose: + * + * Set the end-of-address marker for the file. + * + * Return: + * + * SUCCEED (can't fail) * * Programmer: Jacob Smith * 2017-11-03 * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS - static herr_t H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) { @@ -990,7 +1411,7 @@ H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) FUNC_ENTER_NOAPI_NOINIT_NOERR #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif file->eoa = addr; @@ -998,37 +1419,24 @@ H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) FUNC_LEAVE_NOAPI(SUCCEED) } /* H5FD_hdfs_set_eoa() */ -#else /* H5_HAVE_LIBHDFS not defined */ - -static herr_t -H5FD_hdfs_set_eoa(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t H5_ATTR_UNUSED addr) -{ - FUNC_ENTER_NOAPI_NOINIT_NOERR - -#if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); -#endif - - FUNC_LEAVE_NOAPI(FAIL) -} /* H5FD_hdfs_set_eoa() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_get_eof() * - * Purpose: Returns the end-of-file marker. + * Function: H5FD_hdfs_get_eof() + * + * Purpose: + * + * Returns the end-of-file marker. * - * Return: EOF: the first address past the end of the "file", either the - * filesystem file or the HDF5 file. + * Return: + * + * EOF: the first address past the end of the "file", either the + * filesystem file or the HDF5 file. * * Programmer: Jacob Smith * 2017-11-02 * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS - static haddr_t H5FD_hdfs_get_eof(const H5FD_t *_file) { @@ -1037,7 +1445,7 @@ H5FD_hdfs_get_eof(const H5FD_t *_file) FUNC_ENTER_NOAPI_NOINIT_NOERR #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif HDassert(file->hdfs_handle != NULL); @@ -1046,46 +1454,33 @@ H5FD_hdfs_get_eof(const H5FD_t *_file) FUNC_LEAVE_NOAPI((size_t)file->hdfs_handle->fileinfo->mSize) } /* end H5FD_hdfs_get_eof() */ -#else /* H5_HAVE_LIBHDFS not defined */ - -static haddr_t -H5FD_hdfs_get_eof(const H5FD_t H5_ATTR_UNUSED *_file) -{ - FUNC_ENTER_NOAPI_NOINIT_NOERR - -#if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); -#endif - - FUNC_LEAVE_NOAPI((size_t)0) -} /* end H5FD_hdfs_get_eof() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_get_handle * - * Purpose: Returns the HDFS handle (hdfs_t) of hdfs file driver. + * Function: H5FD_hdfs_get_handle * - * Returns: SUCCEED/FAIL + * Purpose: + * + * Returns the HDFS handle (hdfs_t) of hdfs file driver. + * + * Returns: + * + * SUCCEED/FAIL * * Programmer: Jacob Smith * 2017-11-02 * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS - static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle) { - H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; herr_t ret_value = SUCCEED; + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; FUNC_ENTER_NOAPI_NOINIT #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif /* HDFS_DEBUG */ if (!file_handle) @@ -1097,53 +1492,34 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_get_handle() */ -#else /* H5_HAVE_LIBHDFS not defined */ - -static herr_t -H5FD_hdfs_get_handle(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED fapl, - void H5_ATTR_UNUSED **file_handle) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - -#if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); -#endif /* HDFS_DEBUG */ - - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "Illegal get-handle of unsupported virtual file (hdfs)"); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_hdfs_get_handle() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_read * - * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR - * into buffer BUF according to data transfer properties in DXPL_ID. + * Function: H5FD_hdfs_read + * + * Purpose: + * + * Reads SIZE bytes of data from FILE beginning at address ADDR + * into buffer BUF according to data transfer properties in DXPL_ID. + * + * Return: * - * Return: Success: SUCCEED - * - Result is stored in caller-supplied buffer BUF. - * Failure: FAIL - * - Unable to complete read. - * - Contents of buffer `buf` are undefined. + * Success: `SUCCEED` + * - Result is stored in caller-supplied buffer BUF. + * Failure: `FAIL` + * - Unable to complete read. + * - Contents of buffer `buf` are undefined. * * Programmer: Jacob Smith * 2017-11-?? * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_LIBHDFS - static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, - size_t size, void *buf /*out*/) + size_t size, void *buf) { - H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; herr_t ret_value = SUCCEED; + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; size_t filesize = 0; #if HDFS_STATS /* working variables for storing stats */ @@ -1154,7 +1530,7 @@ H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUS FUNC_ENTER_NOAPI_NOINIT #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif /* HDFS_DEBUG */ HDassert(file != NULL); @@ -1173,13 +1549,10 @@ H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUS #if HDFS_STATS - /* Find which "bin" this read fits in. Can be "overflow" bin. - */ - for (bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) { - if ((unsigned long long)size < hdfs_stats_boundaries[bin_i]) { + /* Find which "bin" this read fits in. Can be "overflow" bin. */ + for (bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) + if ((unsigned long long)size < hdfs_stats_boundaries[bin_i]) break; - } - } bin = (type == H5FD_MEM_DRAW) ? &file->raw[bin_i] : &file->meta[bin_i]; /* Store collected stats in appropriate bin */ @@ -1188,12 +1561,10 @@ H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUS bin->max = size; } else { - if (size < bin->min) { + if (size < bin->min) bin->min = size; - } - if (size > bin->max) { + if (size > bin->max) bin->max = size; - } } bin->count++; bin->bytes += (unsigned long long)size; @@ -1204,35 +1575,18 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_read() */ -#else /* H5_HAVE_LIBHDFS not defined */ - -static herr_t -H5FD_hdfs_read(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, - haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, void H5_ATTR_UNUSED *buf) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - -#if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); -#endif /* HDFS_DEBUG */ - - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "Illegal get-handle of unsupported virtual file (hdfs)"); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_hdfs_read() */ - -#endif /* H5_HAVE_LIBHDFS */ - /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_write * - * Purpose: Write bytes to file. - * UNSUPPORTED IN READ-ONLY HDFS VFD. + * Function: H5FD_hdfs_write + * + * Purpose: + * + * Write bytes to file. + * UNSUPPORTED IN READ-ONLY HDFS VFD. * - * Return: FAIL (Not possible with Read-Only S3 file.) + * Return: + * + * FAIL (Not possible with Read-Only S3 file.) * * Programmer: Jacob Smith * 2017-10-23 @@ -1248,24 +1602,29 @@ H5FD_hdfs_write(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hi FUNC_ENTER_NOAPI_NOINIT #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_write() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot write to read-only file.") + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot write to read-only file") done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_hdfs_write() */ /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_truncate * - * Purpose: Makes sure that the true file size is the same (or larger) - * than the end-of-address. + * Function: H5FD_hdfs_truncate + * + * Purpose: + * + * Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * NOT POSSIBLE ON READ-ONLY S3 FILES. * - * NOT POSSIBLE ON READ-ONLY S3 FILES. + * Return: * - * Return: FAIL (Not possible on Read-Only S3 files.) + * FAIL (Not possible on Read-Only S3 files.) * * Programmer: Jacob Smith * 2017-10-23 @@ -1280,25 +1639,30 @@ H5FD_hdfs_truncate(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED dxpl_id, h FUNC_ENTER_NOAPI_NOINIT #if HDFS_DEBUG - HDfprintf(stdout, "H5FD_hdfs_truncate() called.\n"); + HDfprintf(stdout, "called %s.\n", FUNC); #endif - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot truncate read-only file.") + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot truncate read-only file") done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_hdfs_truncate() */ /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_lock * - * Purpose: Place an advisory lock on a file. - * No effect on Read-Only S3 file. + * Function: H5FD_hdfs_lock * - * Suggestion: remove lock/unlock from class - * would result in error at H5FD_[un]lock() (H5FD.c) + * Purpose: + * + * Place an advisory lock on a file. + * No effect on Read-Only S3 file. + * + * Suggestion: remove lock/unlock from class + * would result in error at H5FD_[un]lock() (H5FD.c) * - * Return: SUCCEED (No-op always succeeds) + * Return: + * + * SUCCEED (No-op always succeeds) * * Programmer: Jacob Smith * 2017-11-03 @@ -1313,12 +1677,17 @@ H5FD_hdfs_lock(H5FD_t H5_ATTR_UNUSED *_file, hbool_t H5_ATTR_UNUSED rw) } /* end H5FD_hdfs_lock() */ /*------------------------------------------------------------------------- - * Function: H5FD_hdfs_unlock * - * Purpose: Remove the existing lock on the file. - * No effect on Read-Only S3 file. + * Function: H5FD_hdfs_unlock + * + * Purpose: + * + * Remove the existing lock on the file. + * No effect on Read-Only S3 file. + * + * Return: * - * Return: SUCCEED (No-op always succeeds) + * SUCCEED (No-op always succeeds) * * Programmer: Jacob Smith * 2017-11-03 @@ -1332,519 +1701,54 @@ H5FD_hdfs_unlock(H5FD_t H5_ATTR_UNUSED *_file) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD_hdfs_unlock() */ -/*------------------------------------------------------------------------- - * Function: H5FD_hdfs_validate_config() - * - * Purpose: Test to see if the supplied instance of H5FD_hdfs_fapl_t - * contains internally consistant data. Return SUCCEED if so, - * and FAIL otherwise. - * - * Note the difference between internally consistant and - * correct. As we will have to try to access the target - * object to determine whether the supplied data is correct, - * we will settle for internal consistancy at this point - * - * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally - * consistant data, FAIL otherwise. - * - * Programmer: Jacob Smith - * 9/10/17 - * - *------------------------------------------------------------------------- +#else /* H5_HAVE_LIBHDFS */ + +/* No-op stubs to avoid binary compatibility problems with previous + * HDF5 1.8 versions. Non-functional versions of these API calls were + * erroneously included in the library even when the HDFS VFD was not + * configured. */ -static herr_t -H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t *fa) +hid_t +H5FD_hdfs_init(void) { - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - - HDassert(fa != NULL); - - if (fa->version != H5FD__CURR_HDFS_FAPL_T_VERSION) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unknown H5FD_hdfs_fapl_t version"); - - if (fa->namenode_port > 65535) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number"); - if (fa->namenode_port < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number"); + /* This should never be called since the header doesn't invoke it */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(H5I_INVALID_HID) +} -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_hdfs_validate_config() */ +void +H5FD_hdfs_term(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI_VOID +} /* end H5FD_hdfs_term() */ -/*------------------------------------------------------------------------- - * Function: H5Pget_fapl_hdfs - * - * Purpose: Returns information about the hdfs file access property - * list though the function arguments. - * - * Return: Success: Non-negative - * Failure: Negative - * - * Programmer: John Mainzer - * 9/10/17 - * - *------------------------------------------------------------------------- - */ herr_t H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out) { - const H5FD_hdfs_fapl_t *fa = NULL; - H5P_genplist_t * plist = NULL; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_API(FAIL) - H5TRACE2("e", "i*x", fapl_id, fa_out); - -#if HDFS_DEBUG - HDfprintf(stdout, "H5Pget_fapl_hdfs() called.\n"); -#endif - - if (fa_out == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL") - plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); - if (plist == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") - if (H5FD_HDFS != H5P_get_driver(plist)) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") - - fa = (const H5FD_hdfs_fapl_t *)H5P_get_driver_info(plist); - if (fa == NULL) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") - - /* Copy the hdfs fapl data out */ - HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); - -done: - FUNC_LEAVE_API(ret_value) -} /* H5Pget_fapl_hdfs() */ - -/*------------------------------------------------------------------------- - * Function: H5FD_hdfs_fapl_get - * - * Purpose: Gets a file access property list which could be used to - * create an identical file. - * - * Return: Success: Ptr to new file access property list value. - * Failure: NULL - * - * Programmer: John Mainzer - * 9/8/17 - * - *------------------------------------------------------------------------- - */ -static void * -H5FD_hdfs_fapl_get(H5FD_t *_file) -{ - H5FD_hdfs_t * file = (H5FD_hdfs_t *)_file; - H5FD_hdfs_fapl_t *fa = NULL; - void * ret_value = NULL; + herr_t ret_value = FAIL; FUNC_ENTER_NOAPI_NOINIT + H5TRACE2("e", "i*x", fapl_id, fa_out); - fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t)); - if (fa == NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* Copy the fields of the structure */ - HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); - - ret_value = fa; + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "HDFS VFD not included in the HDF5 library") done: - if (ret_value == NULL && fa != NULL) { - H5MM_xfree(fa); /* clean up on error */ - } - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_hdfs_fapl_get() */ +} -/*------------------------------------------------------------------------- - * Function: H5FD_hdfs_fapl_copy - * - * Purpose: Copies the hdfs-specific file access properties. - * - * Return: Success: Ptr to a new property list - * Failure: NULL - * - * Programmer: John Mainzer - * 9/8/17 - * - *------------------------------------------------------------------------- - */ -static void * -H5FD_hdfs_fapl_copy(const void *_old_fa) +herr_t +H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa) { - const H5FD_hdfs_fapl_t *old_fa = (const H5FD_hdfs_fapl_t *)_old_fa; - H5FD_hdfs_fapl_t * new_fa = NULL; - void * ret_value = NULL; + herr_t ret_value = FAIL; FUNC_ENTER_NOAPI_NOINIT + H5TRACE2("e", "i*x", fapl_id, fa); - new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t)); - if (new_fa == NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); - ret_value = new_fa; + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "HDFS VFD not included in the HDF5 library") done: - if (ret_value == NULL && new_fa != NULL) - H5MM_xfree(new_fa); /* clean up on error */ - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_hdfs_fapl_copy() */ - -/*------------------------------------------------------------------------- - * Function: H5FD_hdfs_fapl_free - * - * Purpose: Frees the hdfs-specific file access properties. - * - * Return: SUCCEED (cannot fail) - * - * Programmer: John Mainzer - * 9/8/17 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD_hdfs_fapl_free(void *_fa) -{ - H5FD_hdfs_fapl_t *fa = (H5FD_hdfs_fapl_t *)_fa; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - HDassert(fa != NULL); /* sanity check */ - - H5MM_xfree(fa); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FD_hdfs_fapl_free() */ - -#if HDFS_STATS - -/*---------------------------------------------------------------------------- - * Function: hdfs_reset_stats() - * - * Purpose: Reset the stats collection elements in this virtual file structure. - * Clears any set data in stats bins; initializes/zeroes values. - * - * Return: - SUCCESS: `SUCCEED` - * - FAILURE: `FAIL` - * - Occurs if the file is invalid somehow - * - * Programmer: Jacob Smith - * 2017-12-08 - * - *---------------------------------------------------------------------------- - */ -static herr_t -hdfs_reset_stats(H5FD_hdfs_t *file) -{ - unsigned i = 0; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - -#if HDFS_DEBUG - HDprintf("hdfs_reset_stats() called\n"); -#endif - - if (file == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null") - - for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { - file->raw[i].bytes = 0; - file->raw[i].count = 0; - file->raw[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; - file->raw[i].max = 0; - - file->meta[i].bytes = 0; - file->meta[i].count = 0; - file->meta[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; - file->meta[i].max = 0; - } - -done: - FUNC_LEAVE_NOAPI(ret_value); -} /* hdfs_reset_stats */ -#endif /* HDFS_STATS */ - -#if HDFS_STATS - -/*---------------------------------------------------------------------------- - * Function: hdfs_fprint_stats() - * - * Purpose: Tabulate and pretty-print statistics for this virtual file. - * - * Should be called upon file close. - * - * Shows number of reads and bytes read, broken down by - * "raw" (H5FD_MEM_DRAW) - * or "meta" (any other flag) - * - * Prints filename and listing of total number of reads and bytes read, - * both as a grand total and separate meta- and rawdata reads. - * - * If any reads were done, prints out two tables: - * - * 1. overview of raw- and metadata reads - * - min (smallest size read) - * - average of size read - * - k,M,G suffixes by powers of 1024 (2^10) - * - max (largest size read) - * 2. tabulation of "bins", sepraring reads into exponentially-larger - * ranges of size. - * - columns for number of reads, total bytes, and average size, with - * separate sub-colums for raw- and metadata reads. - * - each row represents one bin, identified by the top of its range - * - * Bin ranges can be modified with pound-defines at the top of this file. - * - * Bins without any reads in their bounds are not printed. - * - * An "overflow" bin is also present, to catch "big" reads. - * - * Output for all bins (and range ceiling and average size report) - * is divied by powers of 1024. By corollary, four digits before the decimal - * is valid. - * - * - 41080 bytes is represented by 40.177k, not 41.080k - * - 1004.831M represents approx. 1052642000 bytes - * - * Return: - SUCCESS: `SUCCEED` - * - FAILURE: `FAIL` - * - occurs if the file passed in is invalid - * - TODO: if stream is invalid? how can we check this? - * - * Programmer: Jacob Smith - * - *---------------------------------------------------------------------------- - */ -static herr_t -hdfs_fprint_stats(FILE *stream, const H5FD_hdfs_t *file) -{ - herr_t ret_value = SUCCEED; - parsed_url_t * purl = NULL; - unsigned i = 0; - unsigned long count_meta = 0; - unsigned long count_raw = 0; - double average_meta = 0.0; - double average_raw = 0.0; - unsigned long long min_meta = (unsigned long long)HDFS_STATS_STARTING_MIN; - unsigned long long min_raw = (unsigned long long)HDFS_STATS_STARTING_MIN; - unsigned long long max_meta = 0; - unsigned long long max_raw = 0; - unsigned long long bytes_raw = 0; - unsigned long long bytes_meta = 0; - double re_dub = 0.0; /* re-usable double variable */ - unsigned suffix_i = 0; - const char suffixes[] = {' ', 'K', 'M', 'G', 'T', 'P'}; - - FUNC_ENTER_NOAPI_NOINIT - - if (stream == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file stream cannot be null") - if (file == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file cannot be null") - if (file->hdfs_handle == NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle cannot be null") - if (file->hdfs_handle->magic != HDFS_HDFST_MAGIC) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle has invalid magic") - - /******************* - * AGGREGATE STATS * - *******************/ - - for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { - const hdfs_statsbin *r = &file->raw[i]; - const hdfs_statsbin *m = &file->meta[i]; - - if (m->min < min_meta) { - min_meta = m->min; - } - if (r->min < min_raw) { - min_raw = r->min; - } - if (m->max > max_meta) { - max_meta = m->max; - } - if (r->max > max_raw) { - max_raw = r->max; - } - - count_raw += r->count; - count_meta += m->count; - bytes_raw += r->bytes; - bytes_meta += m->bytes; - } - if (count_raw > 0) { - average_raw = (double)bytes_raw / (double)count_raw; - } - if (count_meta > 0) { - average_meta = (double)bytes_meta / (double)count_meta; - } - - /****************** - * PRINT OVERVIEW * - ******************/ - - HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n", count_raw + count_meta, count_meta, - count_raw); - HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", bytes_raw + bytes_meta, bytes_meta, - bytes_raw); - - if (count_raw + count_meta == 0) { - goto done; - } - - /************************* - * PRINT AGGREGATE STATS * - *************************/ - - HDfprintf(stream, "SIZES meta raw\n"); - HDfprintf(stream, " min "); - if (count_meta == 0) { - HDfprintf(stream, " 0.000 "); - } - else { - re_dub = (double)min_meta; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { - re_dub /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); - } - - if (count_raw == 0) { - HDfprintf(stream, " 0.000 \n"); - } - else { - re_dub = (double)min_raw; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { - re_dub /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); - } - - HDfprintf(stream, " avg "); - re_dub = (double)average_meta; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { - re_dub /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); - - re_dub = (double)average_raw; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { - re_dub /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); - - HDfprintf(stream, " max "); - re_dub = (double)max_meta; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { - re_dub /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); - - re_dub = (double)max_raw; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { - re_dub /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); - - /****************************** - * PRINT INDIVIDUAL BIN STATS * - ******************************/ - - HDfprintf(stream, "BINS # of reads total bytes average size\n"); - HDfprintf(stream, " up-to meta raw meta raw meta raw\n"); - - for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { - const hdfs_statsbin *m; - const hdfs_statsbin *r; - unsigned long long range_end = 0; - char bm_suffix = ' '; /* bytes-meta */ - double bm_val = 0.0; - char br_suffix = ' '; /* bytes-raw */ - double br_val = 0.0; - char am_suffix = ' '; /* average-meta */ - double am_val = 0.0; - char ar_suffix = ' '; /* average-raw */ - double ar_val = 0.0; - - m = &file->meta[i]; - r = &file->raw[i]; - if (r->count == 0 && m->count == 0) { - continue; - } - - range_end = hdfs_stats_boundaries[i]; - - if (i == HDFS_STATS_BIN_COUNT) { - range_end = hdfs_stats_boundaries[i - 1]; - HDfprintf(stream, ">"); - } - else { - HDfprintf(stream, " "); - } - - bm_val = (double)m->bytes; - for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) { - bm_val /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - bm_suffix = suffixes[suffix_i]; - - br_val = (double)r->bytes; - for (suffix_i = 0; br_val >= 1024.0; suffix_i++) { - br_val /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - br_suffix = suffixes[suffix_i]; - - if (m->count > 0) { - am_val = (double)(m->bytes) / (double)(m->count); - } - for (suffix_i = 0; am_val >= 1024.0; suffix_i++) { - am_val /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - am_suffix = suffixes[suffix_i]; - - if (r->count > 0) { - ar_val = (double)(r->bytes) / (double)(r->count); - } - for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) { - ar_val /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); - ar_suffix = suffixes[suffix_i]; - - re_dub = (double)range_end; - for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { - re_dub /= 1024.0; - } - HDassert(suffix_i < sizeof(suffixes)); +} - HDfprintf(stream, " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", re_dub, - suffixes[suffix_i], /* bin ceiling */ - m->count, /* metadata reads */ - r->count, /* rawdata reads */ - bm_val, bm_suffix, /* metadata bytes */ - br_val, br_suffix, /* rawdata bytes */ - am_val, am_suffix, /* metadata average */ - ar_val, ar_suffix); /* rawdata average */ - fflush(stream); - } - -done: - FUNC_LEAVE_NOAPI(ret_value); -} /* hdfs_fprint_stats */ -#endif /* HDFS_STATS */ +#endif /* H5_HAVE_LIBHDFS */ |