diff options
Diffstat (limited to 'tools/lib')
| -rw-r--r-- | tools/lib/h5diff.c | 4 | ||||
| -rw-r--r-- | tools/lib/h5tools.c | 611 | ||||
| -rw-r--r-- | tools/lib/h5tools.h | 49 | ||||
| -rw-r--r-- | tools/lib/h5tools_error.h | 52 | ||||
| -rw-r--r-- | tools/lib/h5tools_utils.c | 159 | ||||
| -rw-r--r-- | tools/lib/h5tools_utils.h | 7 |
6 files changed, 662 insertions, 220 deletions
diff --git a/tools/lib/h5diff.c b/tools/lib/h5diff.c index 86726b4..fff0b9a 100644 --- a/tools/lib/h5diff.c +++ b/tools/lib/h5diff.c @@ -570,14 +570,14 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char *------------------------------------------------------------------------- */ /* open file 1 */ - if((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, (size_t)0)) < 0) { + if((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, H5P_DEFAULT, FALSE, NULL, (size_t)0)) < 0) { parallel_print("h5diff: <%s>: unable to open file\n", fname1); H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname1); } /* end if */ H5TOOLS_DEBUG("file1_id = %s", fname1); /* open file 2 */ - if((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, (size_t)0)) < 0) { + if((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, H5P_DEFAULT, FALSE, NULL, (size_t)0)) < 0) { parallel_print("h5diff: <%s>: unable to open file\n", fname2); H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname2); } /* end if */ diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 33f1e11..57e537d 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -29,7 +29,7 @@ int H5tools_INDENT_g = 0; /* global variables */ -hid_t H5tools_ERR_STACK_g = 0; +hid_t H5tools_ERR_STACK_g = H5I_INVALID_HID; hid_t H5tools_ERR_CLS_g = H5I_INVALID_HID; hid_t H5E_tools_g = H5I_INVALID_HID; hid_t H5E_tools_min_id_g = H5I_INVALID_HID; @@ -54,35 +54,41 @@ unsigned long long packed_data_mask; /* mask in which packed bits to display */ int enable_error_stack = 0; /* re-enable error stack; disable=0 enable=1 */ /* sort parameters */ -H5_index_t sort_by = H5_INDEX_NAME; /*sort_by [creation_order | name] */ -H5_iter_order_t sort_order = H5_ITER_INC; /*sort_order [ascending | descending] */ +H5_index_t sort_by = H5_INDEX_NAME; /* sort_by [creation_order | name] */ +H5_iter_order_t sort_order = H5_ITER_INC; /* sort_order [ascending | descending] */ /* module-scoped variables */ static int h5tools_init_g; /* if h5tools lib has been initialized */ -/* Names of VFDs */ -static const char *drivernames[]={ +/* Names of VOL connectors */ +const char *volnames[] = { + H5VL_NATIVE_NAME, + H5VL_PASSTHRU_NAME, +}; + +/* Names of VFDs. These names are always available so that + * the tools can emit special messages when a VFD is asked + * for by name but is not compiled into the library or is + * somehow otherwise not enabled. + * + */ +const char *drivernames[] = { "sec2", + "direct", + "log", + "windows", + "stdio", + "core", "family", "split", "multi", -#ifdef H5_HAVE_PARALLEL "mpio", -#endif /* H5_HAVE_PARALLEL */ + "ros3", + "hdfs", }; -/* This enum should match the entries in the above drivers_list since they - * are indexes into the drivers_list array. */ -typedef enum { - SEC2_IDX = 0 - ,FAMILY_IDX - ,SPLIT_IDX - ,MULTI_IDX -#ifdef H5_HAVE_PARALLEL - ,MPIO_IDX -#endif /* H5_HAVE_PARALLEL */ -} driver_idx; -#define NUM_DRIVERS (sizeof(drivernames) / sizeof(drivernames[0])) +#define NUM_VOLS (sizeof(volnames) / sizeof(volnames[0])) +#define NUM_DRIVERS (sizeof(drivernames) / sizeof(drivernames[0])) /*------------------------------------------------------------------------- * Function: h5tools_init @@ -131,16 +137,19 @@ h5tools_init(void) void h5tools_close(void) { - H5E_auto2_t tools_func; - void *tools_edata; + H5E_auto2_t tools_func; + void *tools_edata; + if (h5tools_init_g) { /* special case where only data is output to stdout */ - if((rawoutstream == NULL) && rawdatastream && (rawdatastream == stdout)) + if ((rawoutstream == NULL) && rawdatastream && (rawdatastream == stdout)) HDfprintf(rawdatastream, "\n"); H5Eget_auto2(H5tools_ERR_STACK_g, &tools_func, &tools_edata); - if(tools_func!=NULL) + + if (tools_func) H5Eprint2(H5tools_ERR_STACK_g, rawerrorstream); + if (rawattrstream && rawattrstream != stdout) { if (fclose(rawattrstream)) perror("closing rawattrstream"); @@ -425,189 +434,521 @@ h5tools_set_error_file(const char *fname, int is_bin) } /*------------------------------------------------------------------------- - * Function: h5tools_get_fapl + * Function: h5tools_set_vfd_fapl + * + * Purpose: Given a VFL driver name, sets the appropriate driver on the + * specified FAPL. * - * Purpose: Get a FAPL for a given VFL driver name. + * TODO: Add handling for Windows VFD. * * Return: positive - succeeded * negative - failed *------------------------------------------------------------------------- */ -static hid_t -h5tools_get_fapl(hid_t fapl, const char *driver, unsigned *drivernum) +static herr_t +h5tools_set_vfd_fapl(hid_t fapl, h5tools_get_fapl_info_t *get_info) { - hid_t new_fapl = H5I_INVALID_HID; /* Copy of file access property list passed in, or new property list */ herr_t ret_value = SUCCEED; - /* Make a copy of the FAPL, for the file open call to use, eventually */ - if (fapl == H5P_DEFAULT) { - if ((new_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) - H5TOOLS_GOTO_ERROR(FAIL, "H5Pcreate failed"); - } /* end if */ - else { - if ((new_fapl = H5Pcopy(fapl)) < 0) - H5TOOLS_GOTO_ERROR(FAIL, "H5Pcopy failed"); - } /* end else */ - - /* Determine which driver the user wants to open the file with. Try - * that driver. If it can't open it, then fail. */ - if (!HDstrcmp(driver, drivernames[SEC2_IDX])) { - /* SEC2 driver */ - if (H5Pset_fapl_sec2(new_fapl) < 0) + /* Currently, only retrieving a VFD by name is supported */ + if (GET_VFD_BY_NAME != get_info->get_type) + H5TOOLS_GOTO_ERROR(FAIL, "unsupported VFD retrieval type; only retrieval by name is supported"); + + /* + * Determine which driver the user wants to open the file with. + * Try that driver; if it can't open the file, then fail. + */ + if (!HDstrcmp(get_info->u.name, drivernames[SEC2_VFD_IDX])) { + /* SEC2 Driver */ + if (H5Pset_fapl_sec2(fapl) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_sec2 failed"); + } + else if (!HDstrcmp(get_info->u.name, drivernames[DIRECT_VFD_IDX])) { +#ifdef H5_HAVE_DIRECT + /* Direct Driver */ + if (H5Pset_fapl_direct(fapl, 1024, 4096, 8 * 4096) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_direct failed"); +#else + H5TOOLS_GOTO_ERROR(FAIL, "Direct VFD is not enabled"); +#endif + } + else if (!HDstrcmp(get_info->u.name, drivernames[LOG_VFD_IDX])) { + unsigned long long log_flags = H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC; + + /* Log Driver */ + if (H5Pset_fapl_log(fapl, NULL, log_flags, (size_t) 0) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_sec2 failed"); + } + else if (!HDstrcmp(get_info->u.name, drivernames[WINDOWS_VFD_IDX])) { +#ifdef H5_HAVE_WINDOWS - if (drivernum) - *drivernum = SEC2_IDX; +#else + H5TOOLS_GOTO_ERROR(FAIL, "Windows VFD is not enabled"); +#endif + } + else if (!HDstrcmp(get_info->u.name, drivernames[STDIO_VFD_IDX])) { + /* Stdio Driver */ + if (H5Pset_fapl_stdio(fapl) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_stdio failed"); } - else if (!HDstrcmp(driver, drivernames[FAMILY_IDX])) { + else if (!HDstrcmp(get_info->u.name, drivernames[CORE_VFD_IDX])) { + /* Core Driver */ + if (H5Pset_fapl_core(fapl, (size_t) H5_MB, TRUE) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_stdio failed"); + } + else if (!HDstrcmp(get_info->u.name, drivernames[FAMILY_VFD_IDX])) { /* FAMILY Driver */ /* Set member size to be 0 to indicate the current first member size * is the member size. */ - if (H5Pset_fapl_family(new_fapl, (hsize_t) 0, H5P_DEFAULT) < 0) + if (H5Pset_fapl_family(fapl, (hsize_t) 0, H5P_DEFAULT) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_family failed"); - - if (drivernum) - *drivernum = FAMILY_IDX; } - else if (!HDstrcmp(driver, drivernames[SPLIT_IDX])) { + else if (!HDstrcmp(get_info->u.name, drivernames[SPLIT_VFD_IDX])) { /* SPLIT Driver */ - if (H5Pset_fapl_split(new_fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) + if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_split failed"); - - if (drivernum) - *drivernum = SPLIT_IDX; } - else if (!HDstrcmp(driver, drivernames[MULTI_IDX])) { + else if (!HDstrcmp(get_info->u.name, drivernames[MULTI_VFD_IDX])) { /* MULTI Driver */ - if (H5Pset_fapl_multi(new_fapl, NULL, NULL, NULL, NULL, TRUE) < 0) + if (H5Pset_fapl_multi(fapl, NULL, NULL, NULL, NULL, TRUE) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_multi failed"); - - if(drivernum) - *drivernum = MULTI_IDX; } + else if (!HDstrcmp(get_info->u.name, drivernames[MPIO_VFD_IDX])) { #ifdef H5_HAVE_PARALLEL - else if(!HDstrcmp(driver, drivernames[MPIO_IDX])) { int mpi_initialized, mpi_finalized; /* MPI-I/O Driver */ + /* check if MPI is available. */ MPI_Initialized(&mpi_initialized); MPI_Finalized(&mpi_finalized); - if(mpi_initialized && !mpi_finalized) { - if(H5Pset_fapl_mpio(new_fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) + if (mpi_initialized && !mpi_finalized) { + if (H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_mpio failed"); - if(drivernum) - *drivernum = MPIO_IDX; } /* end if */ - } +#else + H5TOOLS_GOTO_ERROR(FAIL, "MPI-I/O VFD is not enabled"); #endif /* H5_HAVE_PARALLEL */ + } + else if (!HDstrcmp(get_info->u.name, drivernames[ROS3_VFD_IDX])) { +#ifdef H5_HAVE_ROS3_VFD + if (!get_info->info) + H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid"); + if (H5Pset_fapl_ros3(fapl, (H5FD_ros3_fapl_t *)get_info->info) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed"); +#else + H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled"); +#endif + } + else if (!HDstrcmp(get_info->u.name, drivernames[HDFS_VFD_IDX])) { +#ifdef H5_HAVE_LIBHDFS + if (!get_info->info) + H5TOOLS_GOTO_ERROR(FAIL, "HDFS VFD info is invalid"); + if (H5Pset_fapl_hdfs(fapl, (H5FD_hdfs_fapl_t *)get_info->info) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_hdfs() failed"); +#else + H5TOOLS_GOTO_ERROR(FAIL, "The HDFS VFD is not enabled"); +#endif + } + else + H5TOOLS_GOTO_ERROR(FAIL, "invalid VFD name"); + +done: + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: h5tools_set_vol_fapl + * + * Purpose: Given a VOL connector name or ID, sets the appropriate + * connector on the specified FAPL. + * + * Return: positive - succeeded + * negative - failed + *------------------------------------------------------------------------- + */ +static herr_t +h5tools_set_vol_fapl(hid_t fapl, h5tools_get_fapl_info_t *get_info) +{ + htri_t connector_is_registered; + hid_t connector_id = H5I_INVALID_HID; + herr_t ret_value = SUCCEED; + + if (GET_VOL_BY_NAME != get_info->get_type && GET_VOL_BY_ID != get_info->get_type) + H5TOOLS_GOTO_ERROR(FAIL, "invalid VOL retrieval type"); + + if (GET_VOL_BY_NAME == get_info->get_type) { + /* Retrieve VOL connector by name */ + + if ((connector_is_registered = H5VLis_connector_registered_by_name(get_info->u.name)) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "can't check if VOL connector is registered"); + if (connector_is_registered) { + if ((connector_id = H5VLget_connector_id_by_name(get_info->u.name)) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "can't get VOL connector ID"); + } + else { + /* Check for VOL connectors that ship with the library */ + if (!strcmp(get_info->u.name, H5VL_NATIVE_NAME)) { + connector_id = H5VL_NATIVE; + } + else if (!strcmp(get_info->u.name, H5VL_PASSTHRU_NAME)) { + connector_id = H5VL_PASSTHRU; + } + else { + if ((connector_id = H5VLregister_connector_by_name(get_info->u.name, H5P_DEFAULT)) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "can't register VOL connector"); + } + } + } + else { + /* Retrieve VOL connector by ID */ + + if ((connector_is_registered = H5VLis_connector_registered_by_value((H5VL_class_value_t) get_info->u.id)) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "can't check if VOL connector is registered"); + if (connector_is_registered) { + if ((connector_id = H5VLget_connector_id_by_value((H5VL_class_value_t) get_info->u.id)) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "can't get VOL connector ID"); + } + else { + /* Check for VOL connectors that ship with the library */ + if (get_info->u.id == H5VL_NATIVE_VALUE) { + connector_id = H5VL_NATIVE; + } + else if (get_info->u.id == H5VL_PASSTHRU_VALUE) { + connector_id = H5VL_PASSTHRU; + } + else { + if ((connector_id = H5VLregister_connector_by_value((H5VL_class_value_t) get_info->u.id, H5P_DEFAULT)) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "can't register VOL connector"); + } + } + } + + if (H5Pset_vol(fapl, connector_id, get_info->info) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "can't set VOL connector on FAPL"); + +done: + if (ret_value < 0) { + if (connector_id >= 0 && H5Idec_ref(connector_id) < 0) + H5TOOLS_ERROR(FAIL, "failed to decrement refcount on VOL connector ID"); + } + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: h5tools_get_fapl + * + * Purpose: Get a FAPL for a given VFL driver name, VOL connector name + * or VOL connector ID. + * + * Return: positive - succeeded + * negative - failed + *------------------------------------------------------------------------- + */ +hid_t +h5tools_get_fapl(hid_t fapl, h5tools_get_fapl_info_t *get_info) +{ + hid_t new_fapl = H5I_INVALID_HID; + hid_t ret_value = H5I_INVALID_HID; + + if (fapl < 0) + H5TOOLS_GOTO_ERROR(FAIL, "invalid FAPL"); + if (!get_info) + H5TOOLS_GOTO_ERROR(FAIL, "invalid FAPL retrieval info"); + + /* Make a copy of the FAPL if necessary, or create a FAPL if + * H5P_DEFAULT is specified. */ + if (H5P_DEFAULT == fapl) { + if ((new_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "H5Pcreate failed"); + } /* end if */ + else { + if ((new_fapl = H5Pcopy(fapl)) < 0) + H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "H5Pcopy failed"); + } + + if (GET_VFD_BY_NAME == get_info->get_type) { + if (h5tools_set_vfd_fapl(new_fapl, get_info) < 0) + H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "failed to set VFD on FAPL"); + } + else if (GET_VOL_BY_NAME == get_info->get_type || GET_VOL_BY_ID == get_info->get_type) { + if (h5tools_set_vol_fapl(new_fapl, get_info) < 0) + H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "failed to set VOL on FAPL"); + } else - ret_value = -1; + H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "invalid FAPL retrieval type"); + + ret_value = new_fapl; done: - if((new_fapl != H5P_DEFAULT) && (ret_value < 0)) { + if ((new_fapl >= 0) && (ret_value < 0)) { H5Pclose(new_fapl); new_fapl = H5I_INVALID_HID; } - return(new_fapl); + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: h5tools_get_vfd_name + * + * Purpose: Given a FAPL, retrieves the name of the VFL driver set on it + * if using a native-terminal VOL connector. If a + * non-native-terminal VOL connector is set on the FAPL, the + * first byte of the returned driver name will be set to the null + * terminator. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +h5tools_get_vfd_name(hid_t fapl_id, char *drivername, size_t drivername_size) +{ + hid_t fapl_vol_id = H5I_INVALID_HID; + herr_t ret_value = SUCCEED; + + if (fapl_id < 0) + H5TOOLS_GOTO_ERROR(FAIL, "invalid FAPL"); + if (!drivername) + H5TOOLS_GOTO_ERROR(FAIL, "drivername is NULL"); + if (drivername && !drivername_size) + H5TOOLS_GOTO_ERROR(FAIL, "drivername_size must be non-zero"); + + /* Initialize the driver name */ + drivername[0] = '\0'; + + if (fapl_id == H5P_DEFAULT) + fapl_id = H5P_FILE_ACCESS_DEFAULT; + + /* Retrieve ID of the VOL connector set on the FAPL */ + if (H5Pget_vol_id(fapl_id, &fapl_vol_id) < 0) + H5TOOLS_ERROR(FAIL, "failed to retrieve VOL ID from FAPL"); + + /* TODO: For now, we have no way of determining if an arbitrary + * VOL connector is native-terminal. */ + if (fapl_vol_id == H5VL_NATIVE) { + const char *driver_name; + hid_t driver_id; + + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "failed to retrieve VFL driver ID from FAPL"); + + if (driver_id == H5FD_SEC2) + driver_name = drivernames[SEC2_VFD_IDX]; +#ifdef H5_HAVE_DIRECT + else if (driver_id == H5FD_DIRECT) + driver_name = drivernames[DIRECT_VFD_IDX]; +#endif + else if (driver_id == H5FD_LOG) + driver_name = drivernames[LOG_VFD_IDX]; +#ifdef H5_HAVE_WINDOWS + else if (driver_id == H5FD_WINDOWS) + driver_name = drivernames[WINDOWS_VFD_IDX]; +#endif + else if (driver_id == H5FD_STDIO) + driver_name = drivernames[STDIO_VFD_IDX]; + else if (driver_id == H5FD_CORE) + driver_name = drivernames[CORE_VFD_IDX]; + else if (driver_id == H5FD_FAMILY) + driver_name = drivernames[FAMILY_VFD_IDX]; + else if (driver_id == H5FD_MULTI) + driver_name = drivernames[MULTI_VFD_IDX]; +#ifdef H5_HAVE_PARALLEL + else if (driver_id == H5FD_MPIO) + driver_name = drivernames[MPIO_VFD_IDX]; +#endif +#ifdef H5_HAVE_ROS3_VFD + else if (driver_id == H5FD_ROS3) + driver_name = drivernames[ROS3_VFD_IDX]; +#endif +#ifdef H5_HAVE_LIBHDFS + else if (driver_id == H5FD_HDFS) + driver_name = drivernames[HDFS_VFD_IDX]; +#endif + else + driver_name = "unknown"; + + HDstrncpy(drivername, driver_name, drivername_size); + drivername[drivername_size - 1] = '\0'; + } + +done: + /* Close retrieved VOL ID */ + if (fapl_vol_id >= 0) + if (H5VLclose(fapl_vol_id) < 0) + H5TOOLS_ERROR(FAIL, "failed to close VOL ID"); + + return ret_value; } /*------------------------------------------------------------------------- * Function: h5tools_fopen * - * Purpose: Loop through the various types of VFL drivers trying to open FNAME. - * If the HDF5 library is version 1.2 or less, then we have only the SEC2 - * driver to try out. If the HDF5 library is greater than version 1.2, - * then we have the FAMILY, SPLIT, and MULTI drivers to play with. + * Purpose: Opens file FNAME using the specified flags and FAPL. + * + * The 'use_specific_driver' parameter is used to control the + * VFD/VOL connector that this routine uses to open the file + * with. If 'use_specific_driver' is set to TRUE, this routine + * assumes that the caller has already set a specific VFD or VOL + * connector on the given FAPL and will attempt to directly use + * the FAPL for opening the file. We assume that the caller knows + * what they are doing; if the file is unable to be opened using + * that FAPL, this routine will return H5I_INVALID_HID. * - * If DRIVER is non-NULL, then it will try to open the file with that - * driver first. We assume that the user knows what they are doing so, if - * we fail, then we won't try other file drivers. + * However, if 'use_specific_driver' is set to FALSE, this + * routine assumes that the caller HAS NOT set a specific VFD or + * VOL connector on the given FAPL and will instead loop through + * the various available VFL drivers and VOL connectors trying to + * open FNAME. + * + * The list of available VFL drivers is as follows: + * - If the HDF5 library is version 1.2 or less, then we have + * only the SEC2 driver to try out. + * - If the HDF5 library is greater than version 1.2, then we + * have the FAMILY, SPLIT, and MULTI drivers to play with. + * + * The list of available VOL connectors is as follows: + * - "Native" VOL connector + * - Pass-through VOL connector * * Return: - * On success, returns a file id for the opened file. If DRIVERNAME is - * non-null then the first DRIVERNAME_SIZE-1 characters of the driver - * name are copied into the DRIVERNAME array and null terminated. + * On success, returns a file ID for the opened file. If DRIVERNAME is + * non-null and the native VOL connector is the terminal connector, + * then the first DRIVERNAME_SIZE-1 characters of the driver name are + * copied into the DRIVERNAME array and null terminated. If the + * native VOL connector is NOT the terminal connector, then the first + * byte of DRIVERNAME will be set to the null terminator. * - * Otherwise, the function returns FAIL. If DRIVERNAME is non-null then - * the first byte is set to the null terminator. + * On failure, the function returns H5I_INVALID_HID and DRIVERNAME + * will not be set. *------------------------------------------------------------------------- */ hid_t -h5tools_fopen(const char *fname, unsigned flags, hid_t fapl, const char *driver, +h5tools_fopen(const char *fname, unsigned flags, hid_t fapl, hbool_t use_specific_driver, char *drivername, size_t drivername_size) { - unsigned drivernum; - hid_t fid = FAIL; - hid_t my_fapl = H5P_DEFAULT; - - if (driver && *driver) { - /* Get the correct FAPL for the given driver */ - if ((my_fapl = h5tools_get_fapl(fapl, driver, &drivernum)) < 0) - goto done; + hid_t fid = H5I_INVALID_HID; + hid_t tmp_vol_fapl = H5I_INVALID_HID; + hid_t tmp_vfd_fapl = H5I_INVALID_HID; + hid_t used_fapl = H5I_INVALID_HID; + unsigned volnum, drivernum; + hid_t ret_value = H5I_INVALID_HID; - /* allow error stack display if enable-error-stack has optional arg number */ - if (enable_error_stack > 1) { - fid = H5Fopen(fname, flags, my_fapl); - } - else { - H5E_BEGIN_TRY { - fid = H5Fopen(fname, flags, my_fapl); - } H5E_END_TRY; - } - - if (fid == FAIL) - goto done; + /* + * First try to open the file using just the given FAPL. If the + * HDF5_VOL_CONNECTOR environment variable has been set, this will + * allow us to attempt to open the file using the specified VOL + * connector before we go looping through all available ones, + * which will override any VOL connector set by use of the + * environment variable. + */ + /* Allow error stack display if --enable-error-stack has optional arg number */ + if (enable_error_stack > 1) { + fid = H5Fopen(fname, flags, fapl); } else { - /* Try to open the file using each of the drivers */ - for (drivernum = 0; drivernum < NUM_DRIVERS; drivernum++) { - /* Get the correct FAPL for the given driver */ - if((my_fapl = h5tools_get_fapl(fapl, drivernames[drivernum], NULL)) < 0) - goto done; - - /* allow error stack display if enable-error-stack has optional arg number */ - if (enable_error_stack > 1) { - fid = H5Fopen(fname, flags, my_fapl); - } - else { - H5E_BEGIN_TRY { - fid = H5Fopen(fname, flags, my_fapl); - } H5E_END_TRY; - } + H5E_BEGIN_TRY { + fid = H5Fopen(fname, flags, fapl); + } H5E_END_TRY; + } - if (fid != FAIL) - break; - else { - /* Close the FAPL */ - H5Pclose(my_fapl); - my_fapl = H5P_DEFAULT; - } /* end else */ - } + /* If we succeeded in opening the file, we're done. */ + if (fid >= 0) { + used_fapl = fapl; + H5TOOLS_GOTO_DONE(fid); } - /* Save the driver name */ - if (drivername && drivername_size) { - if (fid != FAIL) { - HDstrncpy(drivername, drivernames[drivernum], drivername_size); - drivername[drivername_size - 1] = '\0'; + /* + * If we failed to open the file and the caller specified 'use_specific_driver' + * as TRUE, we should return failure now since the file couldn't be opened with + * the VFL driver/VOL connector that was set on the FAPL by the caller. + */ + if (fid < 0 && use_specific_driver) + H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "failed to open file using specified FAPL"); + + /* + * As a final resort, try to open the file using each of the available + * VOL connectors. When the native VOL connector is the current "terminal" + * connector being looked at, also try using each of the available VFL drivers. + */ + for (volnum = 0; volnum < NUM_VOLS; volnum++) { + h5tools_get_fapl_info_t get_vol_info; + + get_vol_info.get_type = GET_VOL_BY_NAME; + get_vol_info.info = NULL; + get_vol_info.u.name = volnames[volnum]; + + /* Get a FAPL for the current VOL connector */ + if ((tmp_vol_fapl = h5tools_get_fapl(fapl, &get_vol_info)) < 0) + continue; + + /* TODO: For now, we have no way of determining if an arbitrary + * VOL connector is native-terminal. */ + if (NATIVE_VOL_IDX == volnum) { + /* + * If using the native VOL connector, or a VOL connector which has the + * native connector as its terminal connector, loop through all of the + * VFL drivers as well. + */ + for (drivernum = 0; drivernum < NUM_DRIVERS; drivernum++) { + h5tools_get_fapl_info_t get_vfd_info; + + /* Skip the log VFD as it prints out to standard out */ + if (drivernum == LOG_VFD_IDX) + continue; + + get_vfd_info.get_type = GET_VFD_BY_NAME; + get_vfd_info.info = NULL; + get_vfd_info.u.name = drivernames[drivernum]; + + /* Using the current VOL FAPL as a base, get the correct FAPL for the given VFL driver */ + if ((tmp_vfd_fapl = h5tools_get_fapl(tmp_vol_fapl, &get_vfd_info)) < 0) + continue; + + if ((fid = h5tools_fopen(fname, flags, tmp_vfd_fapl, TRUE, drivername, drivername_size)) >= 0) { + used_fapl = tmp_vfd_fapl; + H5TOOLS_GOTO_DONE(fid); + } + else { + /* Close the temporary VFD FAPL */ + H5Pclose(tmp_vfd_fapl); + tmp_vfd_fapl = H5I_INVALID_HID; + } + } } else { - /*no file opened*/ - drivername[0] = '\0'; + if ((fid = h5tools_fopen(fname, flags, tmp_vol_fapl, TRUE, drivername, drivername_size)) >= 0) { + used_fapl = tmp_vol_fapl; + H5TOOLS_GOTO_DONE(fid); + } + else { + /* Close the temporary VOL FAPL */ + H5Pclose(tmp_vol_fapl); + tmp_vol_fapl = H5I_INVALID_HID; + } } } + /* File was unable to be opened at all */ + ret_value = H5I_INVALID_HID; + done: - if(my_fapl != H5P_DEFAULT) - H5Pclose(my_fapl); + /* Save the driver name if using a native-terminal VOL connector */ + if (drivername && drivername_size && ret_value >= 0) + if (used_fapl >= 0 && h5tools_get_vfd_name(used_fapl, drivername, drivername_size) < 0) + H5TOOLS_ERROR(H5I_INVALID_HID, "failed to retrieve name of VFD used to open file"); - return fid; + if (tmp_vfd_fapl >= 0) + H5Pclose(tmp_vfd_fapl); + if (tmp_vol_fapl >= 0) + H5Pclose(tmp_vol_fapl); + + return ret_value; } /*------------------------------------------------------------------------- diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index 9440e7a..11b0a1d 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -539,6 +539,52 @@ typedef struct h5tools_context_t { int display_char; /* */ } h5tools_context_t; +typedef enum { + GET_VFD_BY_NAME, + GET_VOL_BY_NAME, + GET_VOL_BY_ID +} h5tools_get_fapl_type_t; + +typedef struct h5tools_get_fapl_info_t { + h5tools_get_fapl_type_t get_type; + + /* Pointer to information to be passed to the driver/connector for its setup */ + const void *info; + + /* Field specifying either the driver's/connector's name, or the driver's/connector's ID */ + union { + const char *name; + long id; + } u; +} h5tools_get_fapl_info_t; + +extern const char *volnames[]; +extern const char *drivernames[]; + +/* This enum should match the entries in the above 'volnames' + * since they are indices into the 'volnames' array. */ +typedef enum { + NATIVE_VOL_IDX = 0, + PASS_THROUGH_VOL_IDX, +} vol_idx; + +/* This enum should match the entries in the above 'drivernames' + * since they are indices into the 'drivernames' array. */ +typedef enum { + SEC2_VFD_IDX = 0, + DIRECT_VFD_IDX, + LOG_VFD_IDX, + WINDOWS_VFD_IDX, + STDIO_VFD_IDX, + CORE_VFD_IDX, + FAMILY_VFD_IDX, + SPLIT_VFD_IDX, + MULTI_VFD_IDX, + MPIO_VFD_IDX, + ROS3_VFD_IDX, + HDFS_VFD_IDX, +} driver_idx; + /* The following include, h5tools_str.h, must be after the * above stucts are defined. There is a dependency in the following * include that hasn't been identified yet. */ @@ -591,8 +637,9 @@ H5TOOLS_DLL int h5tools_set_attr_output_file(const char *fname, int is_bin); H5TOOLS_DLL int h5tools_set_input_file(const char *fname, int is_bin); H5TOOLS_DLL int h5tools_set_output_file(const char *fname, int is_bin); H5TOOLS_DLL int h5tools_set_error_file(const char *fname, int is_bin); +H5TOOLS_DLL hid_t h5tools_get_fapl(hid_t fapl, h5tools_get_fapl_info_t *get_info); H5TOOLS_DLL hid_t h5tools_fopen(const char *fname, unsigned flags, hid_t fapl, - const char *driver, char *drivername, size_t drivername_len); + hbool_t use_specific_driver, char *drivername, size_t drivername_size); H5TOOLS_DLL hid_t h5tools_get_little_endian_type(hid_t type); H5TOOLS_DLL hid_t h5tools_get_big_endian_type(hid_t type); H5TOOLS_DLL htri_t h5tools_detect_vlen(hid_t tid); diff --git a/tools/lib/h5tools_error.h b/tools/lib/h5tools_error.h index 4e2e935..764fb91 100644 --- a/tools/lib/h5tools_error.h +++ b/tools/lib/h5tools_error.h @@ -101,14 +101,16 @@ do { * H5TOOLS_PUSH_ERROR macro, used to push an error to an error stack. Not meant to * be called directly. */ -#define H5TOOLS_PUSH_ERROR(estack_id, err_cls, maj_err_id, min_err_id, ...) \ -do { \ - if (estack_id >= 0 && err_cls >= 0) \ - H5Epush2(estack_id, __FILE__, FUNC, __LINE__, err_cls, maj_err_id, min_err_id, __VA_ARGS__); \ - else { \ - HDfprintf(stderr, __VA_ARGS__); \ - HDfprintf(stderr, "\n"); \ - } \ +#define H5TOOLS_PUSH_ERROR(estack_id, err_cls, maj_err_id, min_err_id, ...) \ +do { \ + if (enable_error_stack > 0) { \ + if (estack_id >= 0 && err_cls >= 0) \ + H5Epush2(estack_id, __FILE__, FUNC, __LINE__, err_cls, maj_err_id, min_err_id, __VA_ARGS__); \ + else { \ + HDfprintf(stderr, __VA_ARGS__); \ + HDfprintf(stderr, "\n"); \ + } \ + } \ } while(0) /* @@ -167,30 +169,30 @@ do { #ifdef H5_TOOLS_DEBUG -#define H5TOOLS_START_DEBUG(...) \ -do { \ - H5tools_INDENT_g += 2; \ +#define H5TOOLS_START_DEBUG(...) \ +do { \ + H5tools_INDENT_g += 2; \ HDfprintf(stderr, "%*sENTER %s:%d in %s()...", H5tools_INDENT_g, "", __FILE__, __LINE__, FUNC); \ - HDfprintf(stderr, __VA_ARGS__); \ - HDfprintf(stderr, "\n"); \ - HDfflush(stderr); \ + HDfprintf(stderr, __VA_ARGS__); \ + HDfprintf(stderr, "\n"); \ + HDfflush(stderr); \ } while(0) -#define H5TOOLS_DEBUG(...) \ -do { \ +#define H5TOOLS_DEBUG(...) \ +do { \ HDfprintf(stderr, "%*s %s:%d in %s()...", H5tools_INDENT_g, "", __FILE__, __LINE__, FUNC); \ - HDfprintf(stderr, __VA_ARGS__); \ - HDfprintf(stderr, "\n"); \ - HDfflush(stderr); \ + HDfprintf(stderr, __VA_ARGS__); \ + HDfprintf(stderr, "\n"); \ + HDfflush(stderr); \ } while(0) -#define H5TOOLS_ENDDEBUG(...) \ -do { \ +#define H5TOOLS_ENDDEBUG(...) \ +do { \ HDfprintf(stderr, "%*sEXIT %s:%d in %s()...", H5tools_INDENT_g, "", __FILE__, __LINE__, FUNC); \ - HDfprintf(stderr, __VA_ARGS__); \ - HDfprintf(stderr, "\n"); \ - H5tools_INDENT_g -= 2; \ - HDfflush(stderr); \ + HDfprintf(stderr, __VA_ARGS__); \ + HDfprintf(stderr, "\n"); \ + H5tools_INDENT_g -= 2; \ + HDfflush(stderr); \ } while(0) #else diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index 45e436c..8a7a4dd 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -1079,6 +1079,55 @@ done: return ret_value; } +#ifdef H5_HAVE_ROS3_VFD +/*---------------------------------------------------------------------------- + * + * Function: h5tools_parse_ros3_fapl_tuple + * + * Purpose: A convenience function that parses a string containing a tuple + * of S3 VFD credential information and then passes the result to + * `h5tools_populate_ros3_fapl()` in order to setup a valid + * configuration for the S3 VFD. + * + * Return: SUCCEED/FAIL + * + *---------------------------------------------------------------------------- + */ +herr_t +h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, + H5FD_ros3_fapl_t *fapl_config_out) +{ + const char *ccred[3]; + unsigned nelems = 0; + char *start = NULL; + char *s3cred_src = NULL; + char **s3cred = NULL; + herr_t ret_value = SUCCEED; + + /* Attempt to parse S3 credentials tuple */ + if (parse_tuple(tuple_str, delim, &s3cred_src, &nelems, &s3cred) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "failed to parse S3 VFD info tuple"); + + /* Sanity-check tuple count */ + if (nelems != 3) + H5TOOLS_GOTO_ERROR(FAIL, "invalid S3 VFD credentials"); + + ccred[0] = (const char *)s3cred[0]; + ccred[1] = (const char *)s3cred[1]; + ccred[2] = (const char *)s3cred[2]; + + if (0 == h5tools_populate_ros3_fapl(fapl_config_out, ccred)) + H5TOOLS_GOTO_ERROR(FAIL, "failed to populate S3 VFD FAPL config"); + +done: + if (s3cred) + HDfree(s3cred); + if (s3cred_src) + HDfree(s3cred_src); + + return ret_value; +} + /*---------------------------------------------------------------------------- * @@ -1142,7 +1191,6 @@ done: * *---------------------------------------------------------------------------- */ -#ifdef H5_HAVE_ROS3_VFD int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values) @@ -1266,73 +1314,72 @@ h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, done: return ret_value; - } /* h5tools_populate_ros3_fapl */ #endif /* H5_HAVE_ROS3_VFD */ - -/*----------------------------------------------------------------------------- - * - * Function: h5tools_set_configured_fapl - * - * Purpose: prepare fapl_id with the given property list, according to - * VFD prototype. +#ifdef H5_HAVE_LIBHDFS +/*---------------------------------------------------------------------------- * - * Return: 0 on failure, 1 on success + * Function: h5tools_parse_hdfs_fapl_tuple * - * Programmer: Jacob Smith - * 2018-05-21 + * Purpose: A convenience function that parses a string containing a tuple + * of HDFS VFD configuration information. * - * Changes: None. + * Return: SUCCEED/FAIL * - *----------------------------------------------------------------------------- + *---------------------------------------------------------------------------- */ -int -h5tools_set_configured_fapl(hid_t fapl_id, - const char vfd_name[], - void *fapl_t_ptr) +herr_t +h5tools_parse_hdfs_fapl_tuple(const char *tuple_str, int delim, + H5FD_hdfs_fapl_t *fapl_config_out) { - int ret_value = 1; - - if (fapl_id < 0) { - return 0; + unsigned long k = 0; + unsigned nelems = 0; + char *props_src = NULL; + char **props = NULL; + herr_t ret_value = SUCCEED; + + /* Attempt to parse HDFS configuration tuple */ + if (parse_tuple(tuple_str, delim, &props_src, &nelems, &props) < 0) + H5TOOLS_GOTO_ERROR(FAIL, "failed to parse HDFS VFD configuration tuple"); + + /* Sanity-check tuple count */ + if (nelems != 5) + H5TOOLS_GOTO_ERROR(FAIL, "invalid HDFS VFD configuration"); + + /* Populate fapl configuration structure with given properties. + * WARNING: No error-checking is done on length of input strings... + * Silent overflow is possible, albeit unlikely. + */ + if (HDstrncmp(props[0], "", 1)) { + HDstrncpy(fapl_config_out->namenode_name, (const char *)props[0], HDstrlen(props[0])); } - - if (!strcmp("", vfd_name)) { - goto done; - -#ifdef H5_HAVE_ROS3_VFD - } else if (!strcmp("ros3", vfd_name)) { - if ((fapl_id == H5P_DEFAULT) || - (fapl_t_ptr == NULL) || - (FAIL == H5Pset_fapl_ros3( - fapl_id, - (H5FD_ros3_fapl_t *)fapl_t_ptr))) - { - ret_value = 0; - goto done; - } -#endif /* H5_HAVE_ROS3_VFD */ - -#ifdef H5_HAVE_LIBHDFS - } else if (!strcmp("hdfs", vfd_name)) { - if ((fapl_id == H5P_DEFAULT) || - (fapl_t_ptr == NULL) || - (FAIL == H5Pset_fapl_hdfs( - fapl_id, - (H5FD_hdfs_fapl_t *)fapl_t_ptr))) - { - ret_value = 0; - goto done; - } -#endif /* H5_HAVE_LIBHDFS */ - - } else { - ret_value = 0; /* unrecognized fapl type "name" */ + if (HDstrncmp(props[1], "", 1)) { + k = strtoul((const char *)props[1], NULL, 0); + if (errno == ERANGE) + H5TOOLS_GOTO_ERROR(FAIL, "supposed port number wasn't"); + fapl_config_out->namenode_port = (int32_t)k; + } + if (HDstrncmp(props[2], "", 1)) { + HDstrncpy(fapl_config_out->kerberos_ticket_cache, (const char *)props[2], HDstrlen(props[2])); + } + if (HDstrncmp(props[3], "", 1)) { + HDstrncpy(fapl_config_out->user_name, (const char *)props[3], HDstrlen(props[3])); + } + if (HDstrncmp(props[4], "", 1)) { + k = HDstrtoul((const char *)props[4], NULL, 0); + if (errno == ERANGE) + H5TOOLS_GOTO_ERROR(FAIL, "supposed buffersize number wasn't"); + fapl_config_out->stream_buffer_size = (int32_t)k; } done: - return ret_value; + if (props) + HDfree(props); + if (props_src) + HDfree(props_src); -} /* h5tools_set_configured_fapl() */ + return ret_value; +} +#endif /* H5_HAVE_LIBHDFS */ diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index 598930c..42144cc 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -173,10 +173,15 @@ H5TOOLS_DLL void h5tools_setprogname(const char*progname); H5TOOLS_DLL int h5tools_getstatus(void); H5TOOLS_DLL void h5tools_setstatus(int d_status); H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void); -H5TOOLS_DLL int h5tools_set_configured_fapl(hid_t fapl_id, const char vfd_name[], void *fapl_t_ptr); #ifdef H5_HAVE_ROS3_VFD +H5TOOLS_DLL herr_t h5tools_parse_ros3_fapl_tuple(const char *tuple_str, int delim, + H5FD_ros3_fapl_t *fapl_config_out); H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, const char **values); #endif /* H5_HAVE_ROS3_VFD */ +#ifdef H5_HAVE_LIBHDFS +H5TOOLS_DLL herr_t h5tools_parse_hdfs_fapl_tuple(const char *tuple_str, int delim, + H5FD_hdfs_fapl_t *fapl_config_out); +#endif #ifdef __cplusplus } |