diff options
Diffstat (limited to 'utils')
26 files changed, 4848 insertions, 0 deletions
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt new file mode 100644 index 0000000..a248ce1 --- /dev/null +++ b/utils/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required (VERSION 3.10) +project (HDF5_UTILS C) + +if (BUILD_TESTING) + add_subdirectory (test) +endif () + +option (HDF5_BUILD_UTILS "Build HDF5 Utils" ON) +if (HDF5_BUILD_UTILS) + add_subdirectory (mirror_vfd) +endif () + +#-- Add the h5dwalk and test executables +if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL) + add_subdirectory(tools) +endif() diff --git a/utils/Makefile.am b/utils/Makefile.am new file mode 100644 index 0000000..cd63db4 --- /dev/null +++ b/utils/Makefile.am @@ -0,0 +1,44 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Utils HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +CONFIG=ordered + +if BUILD_TESTS_CONDITIONAL + TESTUTIL_DIR =test +else + TESTUTIL_DIR= +endif + +if MIRROR_VFD_CONDITIONAL + MIRROR_VFD_DIR = mirror_vfd +else + MIRROR_VFD_DIR= +endif + +if BUILD_TOOLS_CONDITIONAL + TOOLS_DIR =tools +else + TOOLS_DIR= +endif + +# All subdirectories +SUBDIRS=$(MIRROR_VFD_DIR) $(TESTUTIL_DIR) $(TOOLS_DIR) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/mirror_vfd/CMakeLists.txt b/utils/mirror_vfd/CMakeLists.txt new file mode 100644 index 0000000..92212e0 --- /dev/null +++ b/utils/mirror_vfd/CMakeLists.txt @@ -0,0 +1,80 @@ +cmake_minimum_required (VERSION 3.10) +project (HDF5_UTILS_MIRRORVFD C) + +#----------------------------------------------------------------------------- +# Add the mirror_server executable +#----------------------------------------------------------------------------- + +set (mirror_server_SOURCES + ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_remote.c + ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_server.c + ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_writer.c + ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_remote.h +) +add_executable (mirror_server ${mirror_server_SOURCES}) +target_include_directories (mirror_server PRIVATE "${HDF5_UITLS_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (mirror_server STATIC) + target_link_libraries (mirror_server PRIVATE ${HDF5_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (mirror_server SHARED) + target_link_libraries (mirror_server PRIVATE ${HDF5_LIBSH_TARGET}) +endif () +set_target_properties (mirror_server PROPERTIES FOLDER utils) +set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};mirror_server") +set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} mirror_server) + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_UTILS_MIRRORVFD_SERVER_FORMAT mirror_server) +endif () + +#----------------------------------------------------------------------------- +# Add the mirror_server_stop executable +#----------------------------------------------------------------------------- + +set (mirror_server_stop_SOURCES ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_server_stop.c) +add_executable (mirror_server_stop ${mirror_server_stop_SOURCES}) +target_include_directories (mirror_server_stop PRIVATE "${HDF5_UITLS_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (mirror_server_stop STATIC) + target_link_libraries (mirror_server_stop PRIVATE ${HDF5_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (mirror_server_stop SHARED) + target_link_libraries (mirror_server_stop PRIVATE ${HDF5_LIBSH_TARGET}) +endif () +set_target_properties (mirror_server_stop PROPERTIES FOLDER utils) +set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};mirror_server_stop") +set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} mirror_server_stop) + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_UTILS_MIRRORVFD_STOP_FORMAT mirror_server_stop) +endif () + +############################################################################## +############################################################################## +### I N S T A L L A T I O N ### +############################################################################## +############################################################################## + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- +if (HDF5_EXPORTED_TARGETS) + foreach (exec ${H5_DEP_EXECUTABLES}) + INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} utilsapplications) + endforeach () + + install ( + TARGETS + ${H5_DEP_EXECUTABLES} + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT utilsapplications + ) +endif () diff --git a/utils/mirror_vfd/Makefile.am b/utils/mirror_vfd/Makefile.am new file mode 100644 index 0000000..f263c4e --- /dev/null +++ b/utils/mirror_vfd/Makefile.am @@ -0,0 +1,30 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +AM_CPPFLAGS+=-I$(top_srcdir)/src + +bin_PROGRAMS = mirror_server mirror_server_stop + +mirror_server_SOURCES = mirror_server.c mirror_writer.c mirror_remote.c +#mirror_writer_SOURCES = mirror_writer.c mirror_remote.c + +# All programs depend on the hdf5 library +LDADD=$(LIBHDF5) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/mirror_vfd/mirror_remote.c b/utils/mirror_vfd/mirror_remote.c new file mode 100644 index 0000000..8bb9544 --- /dev/null +++ b/utils/mirror_vfd/mirror_remote.c @@ -0,0 +1,209 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Common operations for "remote" processes for the Mirror VFD. + * + * Jacob Smith, 2020-03-06 + */ + +#include "mirror_remote.h" + +#ifdef H5_HAVE_MIRROR_VFD + +/* --------------------------------------------------------------------------- + * Function: mirror_log + * + * Purpose: Write message to the logging stream/file. + * If logging info pointer is NULL, uses logging defaults. + * ---------------------------------------------------------------------------- + */ +void +mirror_log(struct mirror_log_info *info, unsigned int level, const char *format, ...) +{ + FILE * stream = MIRROR_LOG_DEFAULT_STREAM; + unsigned int verbosity = MIRROR_LOG_DEFAULT_VERBOSITY; + hbool_t custom = FALSE; + + if (info != NULL && info->magic == MIRROR_LOG_INFO_MAGIC) { + stream = info->stream; + verbosity = info->verbosity; + custom = TRUE; + } + + if (level == V_NONE) { + return; + } + else if (level <= verbosity) { + if (custom == TRUE && info->prefix[0] != '\0') { + HDfprintf(stream, "%s", info->prefix); + } + + switch (level) { + case (V_ERR): + HDfprintf(stream, "ERROR "); + break; + case (V_WARN): + HDfprintf(stream, "WARNING "); + break; + default: + break; + } + + if (format != NULL) { + va_list args; + HDva_start(args, format); + HDvfprintf(stream, format, args); + HDva_end(args); + } + + HDfprintf(stream, "\n"); + HDfflush(stream); + } /* end if sufficiently verbose to print */ +} /* end mirror_log() */ + +/* --------------------------------------------------------------------------- + * Function: session_log_bytes + * + * Purpose: "Pretty-print" raw binary data to logging stream/file. + * If info pointer is NULL, uses logging defaults. + * ---------------------------------------------------------------------------- + */ +void +mirror_log_bytes(struct mirror_log_info *info, unsigned int level, size_t n_bytes, const unsigned char *buf) +{ + FILE * stream = MIRROR_LOG_DEFAULT_STREAM; + unsigned int verbosity = MIRROR_LOG_DEFAULT_VERBOSITY; + + if (buf == NULL) { + return; + } + + if (info != NULL && info->magic == MIRROR_LOG_INFO_MAGIC) { + stream = info->stream; + verbosity = info->verbosity; + } + + if (level <= verbosity) { + size_t bytes_written = 0; + const unsigned char *b = NULL; + + /* print whole lines */ + while ((n_bytes - bytes_written) >= 32) { + b = buf + bytes_written; /* point to region in buffer */ + HDfprintf(stream, + "%04zX %02X%02X%02X%02X %02X%02X%02X%02X" + " %02X%02X%02X%02X %02X%02X%02X%02X" + " %02X%02X%02X%02X %02X%02X%02X%02X" + " %02X%02X%02X%02X %02X%02X%02X%02X\n", + bytes_written, b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], + b[12], b[13], b[14], b[15], b[16], b[17], b[18], b[19], b[20], b[21], b[22], b[23], + b[24], b[25], b[26], b[27], b[28], b[29], b[30], b[31]); + bytes_written += 32; + } + + /* start partial line */ + if (n_bytes > bytes_written) { + HDfprintf(stream, "%04zX ", bytes_written); + } + + /* partial line blocks */ + while ((n_bytes - bytes_written) >= 4) { + HDfprintf(stream, " %02X%02X%02X%02X", buf[bytes_written], buf[bytes_written + 1], + buf[bytes_written + 2], buf[bytes_written + 3]); + bytes_written += 4; + } + + /* block separator before partial block */ + if (n_bytes > bytes_written) { + HDfprintf(stream, " "); + } + + /* partial block individual bytes */ + while (n_bytes > bytes_written) { + HDfprintf(stream, "%02X", buf[bytes_written++]); + } + + /* end partial line */ + HDfprintf(stream, "\n"); + } /* end if suitably verbose to log */ +} /* end mirror_log_bytes() */ + +/* --------------------------------------------------------------------------- + * Function: mirror_log_init + * + * Purpose: Prepare a loginfo_t structure for use. + * + * Return: Success: Pointer to newly-ceated info. + * Failure: NULL. Either unable to allocate or cannot open file. + * ---------------------------------------------------------------------------- + */ +loginfo_t * +mirror_log_init(char *path, const char *prefix, unsigned int verbosity) +{ + loginfo_t *info = NULL; + + info = (loginfo_t *)HDmalloc(sizeof(loginfo_t)); + if (info != NULL) { + info->magic = MIRROR_LOG_INFO_MAGIC; + info->verbosity = verbosity; + info->stream = MIRROR_LOG_DEFAULT_STREAM; + info->prefix[0] = '\0'; + + if (prefix && *prefix) { + HDstrncpy(info->prefix, prefix, MIRROR_LOG_PREFIX_MAX); + } + + if (path && *path) { + FILE *f = NULL; + f = HDfopen(path, "w"); + if (NULL == f) { + HDfprintf(MIRROR_LOG_DEFAULT_STREAM, "WARN custom logging path could not be opened: %s\n", + path); + info->magic += 1; + HDfree(info); + } + else { + info->stream = f; + } + } + + } /* end if able to allocate */ + + return info; +} /* end mirror_log_init() */ + +/* --------------------------------------------------------------------------- + * Function: mirror_log_term + * + * Purpose: Shut down and clean up a loginfo_t structure. + * + * Return: Success: SUCCEED. Resources released. + * Failure: FAIL. Indeterminite state. + * ---------------------------------------------------------------------------- + */ +herr_t +mirror_log_term(loginfo_t *info) +{ + if (info == NULL || info->magic != MIRROR_LOG_INFO_MAGIC) { + return FAIL; + } + if (info->stream != stderr || info->stream != stdout) { + if (HDfclose(info->stream) < 0) { + return FAIL; + } + } + info->magic += 1; + HDfree(info); + return SUCCEED; +} /* end mirror_log_term() */ + +#endif /* H5_HAVE_MIRROR_VFD */ diff --git a/utils/mirror_vfd/mirror_remote.h b/utils/mirror_vfd/mirror_remote.h new file mode 100644 index 0000000..9415f7f --- /dev/null +++ b/utils/mirror_vfd/mirror_remote.h @@ -0,0 +1,50 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Common definitions for "remote" processes for the Mirror VFD. + * + * Jacob Smith, 2020-03-06 + */ + +#include "hdf5.h" +#include "H5private.h" + +#ifdef H5_HAVE_MIRROR_VFD + +#include "H5FDmirror_priv.h" /* Private header for the mirror VFD */ + +#define V_NONE 0 +#define V_ERR 1 +#define V_WARN 2 +#define V_INFO 3 +#define V_ALL 4 + +#define MIRROR_LOG_DEFAULT_STREAM stdout +#define MIRROR_LOG_DEFAULT_VERBOSITY V_WARN +#define MIRROR_LOG_PREFIX_MAX 79 +#define MIRROR_LOG_INFO_MAGIC 0x569D589A + +typedef struct mirror_log_info { + uint32_t magic; + FILE * stream; + unsigned int verbosity; + char prefix[MIRROR_LOG_PREFIX_MAX + 1]; +} loginfo_t; + +void mirror_log(loginfo_t *info, unsigned int level, const char *format, ...); +void mirror_log_bytes(loginfo_t *info, unsigned int level, size_t n_bytes, const unsigned char *buf); +loginfo_t *mirror_log_init(char *path, const char *prefix, unsigned int verbosity); +int mirror_log_term(loginfo_t *loginfo); + +herr_t run_writer(int socketfd, H5FD_mirror_xmit_open_t *xmit_open); + +#endif /* H5_HAVE_MIRROR_VFD */ diff --git a/utils/mirror_vfd/mirror_server.c b/utils/mirror_vfd/mirror_server.c new file mode 100644 index 0000000..5381d95 --- /dev/null +++ b/utils/mirror_vfd/mirror_server.c @@ -0,0 +1,636 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * "Server" application to associate a Mirror VFD Driver with a Writer. + * + * Server waits on a dedicated port for a Driver to attempt to connect. + * When connection is made, reads a message from the Driver. + * If message is "SHUTDOWN", Server closes connection and terminates. + * Else, if it receives an encoded OPEN xmit (from Driver), the Server forks + * itself; the child becomes a dedicated Writer and maintains connection with + * the Driver instance, and the parent process remains a Server and returns + * to listening for incoming requests. + * Else, the message is not recognized and is ignored. + * + * + * + * mirror_server [args] + * + * Primary server for coordinating mirror VFD connections with the remote + * process. + * + * args: + * --help, -h Print help message and exit. + * --port=N Positive integer for primary listening port. + * --verbosity=N Debugging verbosity + * 0: none + * 1: errors + * 2: details + * 3: all + * --logpath=S File path to direct debugging output, if any. + * Default of none prints output to stdout. + * + */ + +#include "mirror_remote.h" + +#ifdef H5_HAVE_MIRROR_VFD + +#define MAXBUF 2048 /* max buffer length. */ +#define LISTENQ 80 /* max pending mirrorS requests */ +#define DEFAULT_PORT 3000 /* default listening port */ +#define MAX_PORT_LOOPS 20 /* max iteratations through port range */ +#define PORT_LOOP_RETRY_DELAY 1 /* seconds to wait between port scans */ + +/* semi-unique "magic" numbers to sanity-check structure pointers */ +#define OP_ARGS_MAGIC 0xCF074379u +#define SERVER_RUN_MAGIC 0x741B459Au + +/* --------------------------------------------------------------------------- + * Structure: struct op_args + * + * Purpose: Convenience structure for holding arguments from command-line. + * + * `magic` (uint32_t) + * Semi-unique number to help validate a pointer to this struct type. + * Must be OP_ARGS_MAGIC to be considered valid. + * + * `help` (int) + * Flag that the help argument was present in the command line. + * + * `main_port` (int) + * Flag that the help argument was present in the command line. + * + * `verbosity` (int) + * Number between 0 (none) and 4 (all) that controls how much detail + * the program prints as a course of logging. + * + * `log_prepend_serv` (int) + * Flag that the logging messages should have 'S- ' at the start of each + * line. + * + * `log_prepend_type` (int) + * Flag that the logging messages should have the associated verbosity + * level present in the line (e.g., "WARN", "ERROR", or "INFO"). + * + * `log_path` (char *) + * Path string from the command line, giving the absolute path + * for the file for logging output. Can be empty. + * + * `writer_log_path` (char *) + * Path string from the command line, giving the absolute path + * for the file for writer's logging output. Can be empty. + * + * --------------------------------------------------------------------------- + */ +struct op_args { + uint32_t magic; + int help; + int main_port; + unsigned int verbosity; + int log_prepend_serv; + int log_prepend_type; + char log_path[PATH_MAX + 1]; + char writer_log_path[PATH_MAX + 1]; +}; + +/* --------------------------------------------------------------------------- + * Structure: struct server_run + * + * Purpose: Convenience structure for holding information about a server + * in operation. + * + * `magic` (uint32_t) + * Semi-unique number to help validate a pointer to this struct type. + * Must be SERVER_RUN_MAGIC to be considered valid. + * + * `log_stream` (FILE *) + * File handle where logging output is directed. + * By default, is stdout. + * + * `opts` (struct opt_args) + * Contained structure, holds the server's configuration. + * + * `listenfd` (int) + * File descriptor of the listening socket. + * + * --------------------------------------------------------------------------- + */ +struct server_run { + uint32_t magic; + struct op_args opts; + struct mirror_log_info *loginfo; + int listenfd; +}; + +/* --------------------------------------------------------------------------- + * Function: mybzero + * + * Purpose: Introduce bzero without neededing it on the system. + * + * Programmer: Jacob Smith + * 2020-03-30 + * --------------------------------------------------------------------------- + */ +static void +mybzero(void *dest, size_t size) +{ + size_t i = 0; + char * s = NULL; + HDassert(dest); + s = (char *)dest; + for (i = 0; i < size; i++) { + *(s + i) = 0; + } +} /* end mybzero() */ + +/* --------------------------------------------------------------------------- + * Function: usage + * + * Purpose: Print the usage message to stdout. + * --------------------------------------------------------------------------- + */ +static void +usage(void) +{ + HDfprintf(stdout, + "mirror_server [options]\n" + "\n" + "Application for providing Mirror Writer process to " + " Mirror VFD on file-open.\n" + "Listens on a dedicated socket; forks as a Writer upon receipt" + " of a valid OPEN xmit.\n" + "\n" + "Options:\n" + "--help [-h] : Print this help message and quit.\n" + "--logpath=PATH : File path for logging output " + "(default none, to stdout).\n" + "--port=PORT : Primary port (default %d).\n" + "--verbosity=NUM : Debug printing level " + "0..4, (default %d).\n", + DEFAULT_PORT, MIRROR_LOG_DEFAULT_VERBOSITY); +} /* end usage() */ + +/* --------------------------------------------------------------------------- + * Function: parse_args + * + * Purpose: Read command line options and store results in args_out + * structure. Fails in event of unrecognized option. + * + * Return: 0 on success, -1 on failure. + * --------------------------------------------------------------------------- + */ +static int +parse_args(int argc, char **argv, struct op_args *args_out) +{ + int i; + + /* preset default values + */ + args_out->main_port = DEFAULT_PORT; + args_out->help = 0; + args_out->log_prepend_serv = 1; + args_out->log_prepend_type = 1; + args_out->verbosity = MIRROR_LOG_DEFAULT_VERBOSITY; + /* preset empty strings */ + mybzero(args_out->log_path, PATH_MAX + 1); + mybzero(args_out->writer_log_path, PATH_MAX + 1); + + if (argv == NULL || *argv == NULL) { + mirror_log(NULL, V_ERR, "invalid argv pointer"); + return -1; + } + + /* Loop over arguments after program name and writer_path */ + for (i = 2; i < argc; i++) { + if (!HDstrncmp(argv[i], "-h", 3) || !HDstrncmp(argv[i], "--help", 7)) { + mirror_log(NULL, V_INFO, "found help argument"); + args_out->help = 1; + return 0; + } /* end if help */ + else if (!HDstrncmp(argv[i], "--port=", 7)) { + mirror_log(NULL, V_INFO, "parsing 'main_port' (%s)", argv[i] + 7); + args_out->main_port = HDatoi(argv[i] + 7); + } /* end if port */ + else if (!HDstrncmp(argv[i], "--verbosity=", 12)) { + mirror_log(NULL, V_INFO, "parsing 'verbosity' (%s)", argv[i] + 12); + args_out->verbosity = (unsigned int)HDatoi(argv[i] + 12); + } /* end if verbosity */ + else if (!HDstrncmp(argv[i], "--logpath=", 10)) { + mirror_log(NULL, V_INFO, "parsing 'logpath' (%s)", argv[i] + 10); + HDstrncpy(args_out->log_path, argv[i] + 10, PATH_MAX); + } /* end if logpath */ + else { + mirror_log(NULL, V_ERR, "unrecognized argument: %s", argv[i]); + return -1; + } /* end if unrecognized argument */ + } /* end for each arg after the path to writer "receiver process" */ + + mirror_log(NULL, V_INFO, "all args parsed"); + + return 0; +} /* end parse_args() */ + +/* --------------------------------------------------------------------------- + * Function: prepare_listening_socket + * + * Purpose: Configure and open a socket. + * In event of error, attempts to undo its processes. + * + * Return: Success: non-negative (the file descriptor of the socket) + * Failure: -1 + * --------------------------------------------------------------------------- + */ +static int +prepare_listening_socket(struct server_run *run) +{ + struct sockaddr_in server_addr; + int _true = 1; /* needed for setsockopt() */ + int ret_value = -1; + int ret = 0; /* for checking return value of function calls */ + + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + mirror_log(run->loginfo, V_INFO, "preparing socket"); + + server_addr.sin_family = AF_INET; + server_addr.sin_addr.s_addr = HDhtonl(INADDR_ANY); + server_addr.sin_port = HDhtons((uint16_t)run->opts.main_port); + + mirror_log(run->loginfo, V_INFO, "socket()"); + ret_value = HDsocket(AF_INET, SOCK_STREAM, 0); + if (ret_value < 0) { + mirror_log(run->loginfo, V_ERR, "listening socket:%d", ret_value); + goto error; + } + + mirror_log(run->loginfo, V_ALL, "setsockopt()"); + HDsetsockopt(ret_value, SOL_SOCKET, SO_REUSEADDR, &_true, sizeof(int)); + + mirror_log(run->loginfo, V_INFO, "bind()"); + ret = HDbind(ret_value, (struct sockaddr *)&server_addr, sizeof(server_addr)); + if (ret < 0) { + mirror_log(run->loginfo, V_ERR, "bind() %s", HDstrerror(errno)); + goto error; + } + + mirror_log(run->loginfo, V_INFO, "listen()"); + ret = HDlisten(ret_value, LISTENQ); + if (ret < 0) { + mirror_log(run->loginfo, V_ERR, "H5FD server listen:%d", ret); + goto error; + } + + return ret_value; + +error: + if (ret_value >= 0) { + HDshutdown(ret_value, SHUT_RDWR); + HDclose(ret_value); + } + return -1; +} /* end prepare_listening_socket() */ + +/* --------------------------------------------------------------------------- + * Function: init_server_run + * + * Purpose: Set up server_run struct with default and specified values. + * + * Return: Zero (0) if successful, -1 if an error occurred. + * --------------------------------------------------------------------------- + */ +static struct server_run * +init_server_run(int argc, char **argv) +{ + struct server_run *run; + + run = (struct server_run *)HDmalloc(sizeof(struct server_run)); + if (run == NULL) { + mirror_log(NULL, V_ERR, "can't allocate server_run struct"); + return NULL; + } + + run->magic = (uint32_t)SERVER_RUN_MAGIC; + run->opts.magic = (uint32_t)OP_ARGS_MAGIC; + run->listenfd = -1; + + if (parse_args(argc, argv, &(run->opts)) < 0) { + mirror_log(NULL, V_ERR, "can't parse arguments"); + usage(); + goto error; + } + + if (run->opts.help) { + usage(); + return run; /* early exit */ + } + + run->loginfo = mirror_log_init(run->opts.log_path, "s- ", run->opts.verbosity); + + run->listenfd = prepare_listening_socket(run); + if (run->listenfd < 0) { + mirror_log(NULL, V_ERR, "can't prepare listening socket"); + goto error; + } + + return run; + +error: + if (run != NULL) { + HDfree(run); + } + return NULL; + +} /* end init_server_run() */ + +/* --------------------------------------------------------------------------- + * Function: term_server_run + * + * Purpose: Close opened items in a sever_run and release the pointer. + * + * Return: Zero (0) if successful, -1 if an error occurred. + * --------------------------------------------------------------------------- + */ +static int +term_server_run(struct server_run *run) +{ + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + mirror_log(run->loginfo, V_INFO, "shutting down"); + + if (run->listenfd >= 0) { + HDshutdown(run->listenfd, SHUT_RDWR); /* TODO: error-checking? */ + HDclose(run->listenfd); /* TODO: error-checking? */ + run->listenfd = -1; + } + + if (mirror_log_term(run->loginfo) < 0) { + mirror_log(NULL, V_ERR, "can't close logging stream"); + return -1; /* doesn't solve the problem, but informs of error */ + } + run->loginfo = NULL; + + (run->magic)++; + (run->opts.magic)++; + HDfree(run); + return 0; +} /* end term_server_run() */ + +/* --------------------------------------------------------------------------- + * Function: accept_connection + * + * Purpose: Main working loop; process requests as they are received. + * Does nothing if the run option help is set. + * + * Return: -1 on error, else a non-negative file descriptor of the socket. + * --------------------------------------------------------------------------- + */ +static int +accept_connection(struct server_run *run) +{ + struct sockaddr_in client_addr; /**/ + socklen_t clilen; /**/ + struct hostent * host_port = NULL; /**/ + char * hostaddrp; /**/ + int connfd = -1; /* connection file descriptor */ + + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + /*------------------------------*/ + /* accept a connection on a socket */ + clilen = sizeof(client_addr); + connfd = HDaccept(run->listenfd, (struct sockaddr *)&client_addr, &clilen); + if (connfd < 0) { + mirror_log(run->loginfo, V_ERR, "accept:%d", connfd); + goto error; + } + mirror_log(run->loginfo, V_INFO, "connection achieved"); + + /*------------------------------*/ + /* get client address information */ + host_port = HDgethostbyaddr((const char *)&client_addr.sin_addr.s_addr, + sizeof(client_addr.sin_addr.s_addr), AF_INET); + if (host_port == NULL) { + mirror_log(run->loginfo, V_ERR, "gethostbyaddr()"); + goto error; + } + + /* function has the string space statically scoped -- OK until next call */ + hostaddrp = HDinet_ntoa(client_addr.sin_addr); + /* TODO? proper error-checking */ + + mirror_log(run->loginfo, V_INFO, "server connected with %s (%s)", host_port->h_name, hostaddrp); + + return connfd; + +error: + if (connfd >= 0) { + close(connfd); + } + return -1; +} /* end accept_connection() */ + +/* --------------------------------------------------------------------------- + * Function: wait_for_child + * + * Purpose: Signal handler to reap zombie processes. + * --------------------------------------------------------------------------- + */ +static void +wait_for_child(int H5_ATTR_UNUSED sig) +{ + while (HDwaitpid(-1, NULL, WNOHANG) > 0) + ; +} /* end wait_for_child() */ + +/* --------------------------------------------------------------------------- + * Function: handle_requests + * + * Purpose: Main working loop; process requests as they are received. + * Does nothing if the run option `help` is set. + * + * Return: -1 on error, else 0 for successful operation. + * --------------------------------------------------------------------------- + */ +static int +handle_requests(struct server_run *run) +{ + int connfd = -1; /**/ + char mybuf[H5FD_MIRROR_XMIT_OPEN_SIZE]; /**/ + ssize_t ret; /* general-purpose error-checking */ + int pid; /* process ID of fork */ + struct sigaction sa; + int ret_value = 0; + + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + if (run->opts.help) { + return 0; + } + + if (run->listenfd < 0) { + mirror_log(NULL, V_ERR, "invalid listening socket"); + return -1; + } + + /* Set up the signal handler */ + sa.sa_handler = wait_for_child; + HDsigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + if (HDsigaction(SIGCHLD, &sa, NULL) == -1) { + perror("sigaction"); + return 1; + } + + /* Keep listening for attempts to connect. + */ + + while (1) { /* infinite loop, exited via break or goto */ + mirror_log(run->loginfo, V_INFO, "server waiting for connections..."); + + connfd = -1; + + connfd = accept_connection(run); + if (connfd < 0) { + mirror_log(run->loginfo, V_ERR, "unable to receive connection"); + goto error; + } + + /* Read handshake from port connection. + */ + + if ((ret = HDread(connfd, &mybuf, H5FD_MIRROR_XMIT_OPEN_SIZE)) < 0) { + mirror_log(run->loginfo, V_ERR, "read:%d", ret); + goto error; + } + mirror_log(run->loginfo, V_INFO, "received %d bytes", ret); + mirror_log(run->loginfo, V_ALL, "```"); + mirror_log_bytes(run->loginfo, V_ALL, (size_t)ret, (const unsigned char *)mybuf); + mirror_log(run->loginfo, V_ALL, "```"); + + /* Respond to handshake message. + */ + + if (!HDstrncmp("SHUTDOWN", mybuf, 8)) { + /* Stop operation if told to stop */ + mirror_log(run->loginfo, V_INFO, "received SHUTDOWN!", ret); + HDclose(connfd); + connfd = -1; + goto done; + } /* end if explicit "SHUTDOWN" directive */ + else if (H5FD_MIRROR_XMIT_OPEN_SIZE == ret) { + H5FD_mirror_xmit_open_t xopen; + + mirror_log(run->loginfo, V_INFO, "probable OPEN xmit received"); + + H5FD_mirror_xmit_decode_open(&xopen, (const unsigned char *)mybuf); + if (FALSE == H5FD_mirror_xmit_is_open(&xopen)) { + mirror_log(run->loginfo, V_WARN, "expected OPEN xmit was malformed"); + HDclose(connfd); + continue; + } + + mirror_log(run->loginfo, V_INFO, "probable OPEN xmit confirmed"); + + pid = HDfork(); + if (pid < 0) { /* fork error */ + mirror_log(run->loginfo, V_ERR, "cannot fork"); + goto error; + } /* end if fork error */ + else if (pid == 0) { /* child process (writer side of fork) */ + mirror_log(run->loginfo, V_INFO, "executing writer"); + if (run_writer(connfd, &xopen) < 0) { + HDprintf("can't run writer\n"); + } + else { + HDprintf("writer OK\n"); + } + HDclose(connfd); + + HDexit(EXIT_SUCCESS); + } /* end if writer side of fork */ + else { /* parent process (server side of fork) */ + mirror_log(run->loginfo, V_INFO, "tidying up from handshake"); + HDclose(connfd); + } /* end if server side of fork */ + + } /* end else-if valid request for service */ + else { + /* Ignore unrecognized messages */ + HDclose(connfd); + continue; + } /* end else (not a valid message, to be ignored) */ + + } /* end while listening for new connections */ + +done: + if (connfd >= 0) { + mirror_log(run->loginfo, V_WARN, "connfd still open upon cleanup"); + HDclose(connfd); + } + + return ret_value; + +error: + if (connfd >= 0) { + HDclose(connfd); + } + return -1; +} /* end handle_requests() */ + +/* ------------------------------------------------------------------------- */ +int +main(int argc, char **argv) +{ + struct server_run *run; + + run = init_server_run(argc, argv); + if (NULL == run) { + mirror_log(NULL, V_ERR, "can't initialize run"); + HDexit(EXIT_FAILURE); + } + + if (handle_requests(run) < 0) { + mirror_log(run->loginfo, V_ERR, "problem handling requests"); + } + + if (term_server_run(run) < 0) { + mirror_log(NULL, V_ERR, "problem closing server run"); + HDexit(EXIT_FAILURE); + } + + HDexit(EXIT_SUCCESS); +} /* end main() */ + +#else /* H5_HAVE_MIRROR_VFD */ + +int +main(void) +{ + HDprintf("Mirror VFD was not built -- cannot launch server.\n"); + HDexit(EXIT_FAILURE); +} + +#endif /* H5_HAVE_MIRROR_VFD */ diff --git a/utils/mirror_vfd/mirror_server_stop.c b/utils/mirror_vfd/mirror_server_stop.c new file mode 100644 index 0000000..024b33a --- /dev/null +++ b/utils/mirror_vfd/mirror_server_stop.c @@ -0,0 +1,202 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Stop the mirror server + * Exists for cross-platform, optionally remote shutdown. + */ + +#include "H5private.h" /* System compatibility call-wrapper macros */ + +#ifdef H5_HAVE_MIRROR_VFD + +#define MSHS_OPTS_MAGIC 0x613B1C15u /* sanity-checking constant */ +#define MSHS_IP_STR_SIZE 20 +#define MSHS_DEFAULT_IP "127.0.0.1" +#define MSHS_DEFAULT_PORTNO 3000 + +/* ---------------------------------------------------------------------------- + * Structure: struct mshs_opts + * + * Purpose: Convenience structure to hold options as parsed from the + * command line. + * + * `magic` (uint32_t) + * Semi-unique constant to help verify pointer integrity. + * + * `help` (int) + * Flag that the help argument was present. + * + * `portno` (int) + * Port number, as received from arguments. + * + * `ip` (char *) + * IP address string as received from arguments. + * + * ---------------------------------------------------------------------------- + */ +struct mshs_opts { + uint32_t magic; + int help; + int portno; + char ip[MSHS_IP_STR_SIZE + 1]; +}; + +/* ---------------------------------------------------------------------------- + * Function: usage + * + * Purpose: Print usage message to stdout. + * ---------------------------------------------------------------------------- + */ +static void +usage(void) +{ + HDprintf("mirror_server_halten_sie [options]\n" + "System-independent Mirror Server shutdown program.\n" + "Sends shutdown message to Mirror Server at given IP:port\n" + "\n" + "Options:\n" + " -h | --help Print this usage message and exit.\n" + " --ip=ADDR IP Address of remote server (default %s)\n" + " --port=PORT Handshake port of remote server (default %d)\n", + MSHS_DEFAULT_IP, MSHS_DEFAULT_PORTNO); +} /* end usage() */ + +/* ---------------------------------------------------------------------------- + * Function: parse_args + * + * Purpose: Parse command-line arguments, populating the options struct + * pointer as appropriate. + * Default values will be set for unspecified options. + * + * Return: 0 on success, negative (-1) if error. + * ---------------------------------------------------------------------------- + */ +static int +parse_args(int argc, char **argv, struct mshs_opts *opts) +{ + int i = 0; + + opts->magic = MSHS_OPTS_MAGIC; + opts->help = 0; + opts->portno = MSHS_DEFAULT_PORTNO; + HDstrncpy(opts->ip, MSHS_DEFAULT_IP, MSHS_IP_STR_SIZE); + + for (i = 1; i < argc; i++) { /* start with first possible option argument */ + if (!HDstrncmp(argv[i], "-h", 3) || !HDstrncmp(argv[i], "--help", 7)) { + opts->help = 1; + } + else if (!HDstrncmp(argv[i], "--ip=", 5)) { + HDstrncpy(opts->ip, argv[i] + 5, MSHS_IP_STR_SIZE); + } + else if (!HDstrncmp(argv[i], "--port=", 7)) { + opts->portno = HDatoi(argv[i] + 7); + } + else { + HDprintf("Unrecognized option: '%s'\n", argv[i]); + usage(); + opts->magic++; /* invalidate for sanity */ + return -1; + } + } /* end for each argument from command line */ + + /* auto-replace 'localhost' with numeric IP */ + if (!HDstrncmp(opts->ip, "localhost", 10)) { /* include null terminator */ + HDstrncpy(opts->ip, "127.0.0.1", MSHS_IP_STR_SIZE); + } + + return 0; +} /* end parse_args() */ + +/* ---------------------------------------------------------------------------- + * Function: send_shutdown + * + * Purpose: Create socket and send shutdown signal to remote server. + * + * Return: 0 on success, negative (-1) if error. + * ---------------------------------------------------------------------------- + */ +static int +send_shutdown(struct mshs_opts *opts) +{ + int live_socket; + struct sockaddr_in target_addr; + + if (opts->magic != MSHS_OPTS_MAGIC) { + HDprintf("invalid options structure\n"); + return -1; + } + + live_socket = HDsocket(AF_INET, SOCK_STREAM, 0); + if (live_socket < 0) { + HDprintf("ERROR socket()\n"); + return -1; + } + + target_addr.sin_family = AF_INET; + target_addr.sin_port = HDhtons((uint16_t)opts->portno); + target_addr.sin_addr.s_addr = HDinet_addr(opts->ip); + HDmemset(target_addr.sin_zero, '\0', sizeof(target_addr.sin_zero)); + + if (HDconnect(live_socket, (struct sockaddr *)&target_addr, (socklen_t)sizeof(target_addr)) < 0) { + HDprintf("ERROR connect() (%d)\n%s\n", errno, HDstrerror(errno)); + return -1; + } + + if (HDwrite(live_socket, "SHUTDOWN", 9) == -1) { + HDprintf("ERROR write() (%d)\n%s\n", errno, HDstrerror(errno)); + return -1; + } + + if (HDclose(live_socket) < 0) { + HDprintf("ERROR close() can't close socket\n"); + return -1; + } + + return 0; +} /* end send_shutdown() */ + +/* ------------------------------------------------------------------------- */ +int +main(int argc, char **argv) +{ + struct mshs_opts opts; + + if (parse_args(argc, argv, &opts) < 0) { + HDprintf("Unable to parse arguments\n"); + HDexit(EXIT_FAILURE); + } + + if (opts.help) { + usage(); + HDexit(EXIT_FAILURE); + } + + if (send_shutdown(&opts) < 0) { + HDprintf("Unable to send shutdown command\n"); + HDexit(EXIT_FAILURE); + } + + HDexit(EXIT_SUCCESS); +} /* end main() */ + +#else /* H5_HAVE_MIRROR_VFD */ + +/* ------------------------------------------------------------------------- */ +int +main(void) +{ + HDprintf("Mirror VFD not built -- unable to perform shutdown.\n"); + HDexit(EXIT_FAILURE); +} + +#endif /* H5_HAVE_MIRROR_VFD */ diff --git a/utils/mirror_vfd/mirror_writer.c b/utils/mirror_vfd/mirror_writer.c new file mode 100644 index 0000000..5726db5 --- /dev/null +++ b/utils/mirror_vfd/mirror_writer.c @@ -0,0 +1,1048 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Remote writer process for the mirror (socket) VFD. + * + * Writer is started with arguments for slaved port. + * Awaits a connection on the socket. + * Handles instructions from the master 'Driver' process. + * + * Current implementation uses Sec2 as the underlying driver when opening a + * file. This is reflected in the source (H5FDmirror.c) of the Mirror driver. + */ + +#include "mirror_remote.h" + +#ifdef H5_HAVE_MIRROR_VFD + +#define HEXDUMP_XMITS 1 /* Toggle whether to print xmit bytes-blob */ + /* in detailed logging */ +#define HEXDUMP_WRITEDATA 0 /* Toggle whether to print bytes to write */ + /* in detailed logging */ +#define LISTENQ 80 /* max pending Driver requests */ + +#define MW_SESSION_MAGIC 0x88F36B32u +#define MW_SOCK_COMM_MAGIC 0xDF10A157u +#define MW_OPTS_MAGIC 0x3BA8B462u + +/* --------------------------------------------------------------------------- + * Structure: struct mirror_session + * + * Bundle of information used to manage the operation of this remote Writer + * in a "session" with the Driver process. + * + * magic (uint32_t) + * Semi-unique "magic" number used to sanity-check a structure for + * validity. MUST equal MW_SESSION_MAGIC to be valid. + * + * sockfd (int) + * File descriptor to the socket. + * Used for receiving bytes from and writing bytes to the Driver + * across the network. + * If not NULL, should be a valid descriptor. + * + * token (uint32t) + * Number used to help sanity-check received transmission from the Writer. + * Each Driver/Writer pairing should have a semi-unique "token" to help + * guard against commands from the wrong entity. + * + * xmit_count (uint32_t) + * Record of trasmissions received from the Driver. While the transmission + * protocol should be trustworthy, this serves as an additional guard. + * Starts a 0 and should be incremented for each one-way transmission. + * + * file (H5FD_t *) + * Virtual File handle for the hdf5 file. + * Set on file open if H5Fopen() is successful. If NULL, it is invalid. + * + * log_verbosity (unsigned int) + * The verbosity level for logging. Should be set to one of the values + * defined at the top of this file. + * + * log_stream (FILE *) + * File pointer to which logging output is written. Starts (and ends) + * with a default stream, such as stdout, but can be overridden at + * runtime. + * + * reply (H5FD_mirror_xmit_reply_t) + * Structure space for persistent reply data. + * Should be initialized with basic header info (magic, version, op), + * then with session info (token, xmit count), and finally with specific + * reply info (update xmit_count, status code, and message) before + * transmission. + * + * ---------------------------------------------------------------------------- + */ +struct mirror_session { + uint32_t magic; + int sockfd; + uint32_t token; + uint32_t xmit_count; + H5FD_t * file; + loginfo_t * loginfo; + H5FD_mirror_xmit_reply_t reply; +}; + +/* --------------------------------------------------------------------------- + * Structure: struct sock_comm + * + * Structure for placing the data read and pre-processed from Driver in an + * organized fashion. Useful for pre-processing a received xmit. + * + * magic (uint32_t) + * Semi-unique number to sanity-check structure pointer and validity. + * Must equal MW_SOCK_COMM_MAGIC to be valid. + * + * recd_die (int) + * "Boolean" flag indicating that an explicit shutdown/kill/die command + * was received. Potentially useful for debugging and or "manual" + * operation of the program. + * 0 indicates normal operation, non-0 (1) indicates to die. + * + * xmit_recd (H5FD_mirror_xmit_t *) + * Structure pointer for the "xmit header" as decoded from the raw + * binary stream read from the socket. + * + * raw (char *) + * Pointer to a raw byte array, for storing data as read from the + * socket. Bytes buffer is decoded into xmit_t header and derivative + * structures. + * + * raw_size (size_t) + * Give the size of the `raw` buffer. + * + * --------------------------------------------------------------------------- + */ +struct sock_comm { + uint32_t magic; + int recd_die; + H5FD_mirror_xmit_t *xmit_recd; + char * raw; + size_t raw_size; +}; + +/* --------------------------------------------------------------------------- + * Structure: struct mirror_writer_opts + * + * Container for default values and options as parsed from the command line. + * Currently rather vestigal, but may be expanded and/or moved to be set by + * Server and passed around as an argument. + * + * magic (uint32_t) + * Semi-unique number to sanity-check structure pointer and validity. + * Must equal MW_OPTS_MAGIC to be valid. + * + * logpath (char *) + * String pointer. Allocated at runtime. + * Specifies file location for logging output. + * May be NULL -- uses default output (e.g., stdout). + * + * ---------------------------------------------------------------------------- + */ +struct mirror_writer_opts { + uint32_t magic; + char * logpath; +}; + +static void mybzero(void *dest, size_t size); + +static int do_open(struct mirror_session *session, const H5FD_mirror_xmit_open_t *xmit_open); + +/* --------------------------------------------------------------------------- + * Function: mybzero + * + * Purpose: Introduce bzero without neededing it on the system. + * + * Programmer: Jacob Smith + * 2020-03-30 + * --------------------------------------------------------------------------- + */ +static void +mybzero(void *dest, size_t size) +{ + size_t i = 0; + char * s = NULL; + HDassert(dest); + s = (char *)dest; + for (i = 0; i < size; i++) { + *(s + i) = 0; + } +} /* end mybzero() */ + +/* --------------------------------------------------------------------------- + * Function: session_init + * + * Purpose: Populate mirror_session structure with default and + * options-drived values. + * + * Return: An allocated mirror_session structure pointer on success, + * else NULL. + * ---------------------------------------------------------------------------- + */ +static struct mirror_session * +session_init(struct mirror_writer_opts *opts) +{ + struct mirror_session *session = NULL; + + mirror_log(NULL, V_INFO, "session_init()"); + + if (NULL == opts || opts->magic != MW_OPTS_MAGIC) { + mirror_log(NULL, V_ERR, "invalid opts pointer"); + goto error; + } + + session = (struct mirror_session *)HDmalloc(sizeof(struct mirror_session)); + if (session == NULL) { + mirror_log(NULL, V_ERR, "can't allocate session structure"); + goto error; + } + + session->magic = MW_SESSION_MAGIC; + session->sockfd = -1; + session->xmit_count = 0; + session->token = 0; + session->file = NULL; + + session->reply.pub.magic = H5FD_MIRROR_XMIT_MAGIC; + session->reply.pub.version = H5FD_MIRROR_XMIT_CURR_VERSION; + session->reply.pub.op = H5FD_MIRROR_OP_REPLY; + session->reply.pub.session_token = 0; + mybzero(session->reply.message, H5FD_MIRROR_STATUS_MESSAGE_MAX); + + /* Options-derived population + */ + + session->loginfo = mirror_log_init(opts->logpath, "W- ", MIRROR_LOG_DEFAULT_VERBOSITY); + + return session; + +error: + if (session) { + HDfree(session); + } + return NULL; +} /* end session_init() */ + +/* --------------------------------------------------------------------------- + * Function: session_stop + * + * Purpose: Stop and clean up a session. + * Only do this as part of program termination or aborting startup. + * + * Return: 0 on success, or negative sum of errors encountered. + * ---------------------------------------------------------------------------- + */ +static int +session_stop(struct mirror_session *session) +{ + int ret_value = 0; + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "session_stop()"); + + /* Close HDF5 file if it is still open (probably in error) */ + if (session->file) { + mirror_log(session->loginfo, V_WARN, "HDF5 file still open at cleanup"); + if (H5FDclose(session->file) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDclose() during cleanup!"); + ret_value--; + } + } + + /* Socket will be closed by parent side of server fork after exit */ + + /* Close custom logging stream */ + if (mirror_log_term(session->loginfo) < 0) { + mirror_log(NULL, V_ERR, "Problem closing logging stream"); + ret_value--; + } + session->loginfo = NULL; + + /* Invalidate and release structure */ + session->magic++; + HDfree(session); + + return ret_value; +} /* end session_stop() */ + +/* --------------------------------------------------------------------------- + * Function: session_start + * + * Purpose: Initiate session, open files. + * + * Return: Success: A valid mirror_session pointer which must later be + * cleaned up with session_stop(). + * Failure: NULL, after cleaning up after itself. + * --------------------------------------------------------------------------- + */ +static struct mirror_session * +session_start(int socketfd, const H5FD_mirror_xmit_open_t *xmit_open) +{ + struct mirror_session * session = NULL; + struct mirror_writer_opts opts; +#if 0 /* TODO: behaviro option */ + char logpath[H5FD_MIRROR_XMIT_FILEPATH_MAX] = ""; +#endif + + mirror_log(NULL, V_INFO, "session_start()"); + + if (FALSE == H5FD_mirror_xmit_is_open(xmit_open)) { + mirror_log(NULL, V_ERR, "invalid OPEN xmit"); + return NULL; + } + + opts.magic = MW_OPTS_MAGIC; +#if 0 /* TODO: behavior option */ + HDsnprintf(logpath, H5FD_MIRROR_XMIT_FILEPATH_MAX, "%s.log", + xmit_open->filename); + opts.logpath = logpath; +#else + opts.logpath = NULL; +#endif + + session = session_init(&opts); + if (NULL == session) { + mirror_log(NULL, V_ERR, "can't instantiate session"); + goto error; + } + + session->sockfd = socketfd; + + if (do_open(session, xmit_open) < 0) { + mirror_log(NULL, V_ERR, "unable to open file"); + goto error; + } + + return session; + +error: + if (session != NULL) { + if (session_stop(session) < 0) { + mirror_log(NULL, V_WARN, "Can't abort session init"); + } + session = NULL; + } + return NULL; +} + +/* --------------------------------------------------------------------------- + * Function: _xmit_reply + * + * Purpose: Common operations to send a reply xmit through the session. + * + * Return: 0 on success, -1 if error. + * ---------------------------------------------------------------------------- + */ +static int +_xmit_reply(struct mirror_session *session) +{ + unsigned char xmit_buf[H5FD_MIRROR_XMIT_REPLY_SIZE]; + H5FD_mirror_xmit_reply_t *reply = &(session->reply); + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_ALL, "_xmit_reply()"); + + reply->pub.xmit_count = session->xmit_count++; + if (H5FD_mirror_xmit_encode_reply(xmit_buf, (const H5FD_mirror_xmit_reply_t *)reply) != + H5FD_MIRROR_XMIT_REPLY_SIZE) { + mirror_log(session->loginfo, V_ERR, "can't encode reply"); + return -1; + } + + mirror_log(session->loginfo, V_ALL, "reply xmit data\n```"); + mirror_log_bytes(session->loginfo, V_ALL, H5FD_MIRROR_XMIT_REPLY_SIZE, (const unsigned char *)xmit_buf); + mirror_log(session->loginfo, V_ALL, "```"); + + if (HDwrite(session->sockfd, xmit_buf, H5FD_MIRROR_XMIT_REPLY_SIZE) < 0) { + mirror_log(session->loginfo, V_ERR, "can't write reply to Driver"); + return -1; + } + + return 0; +} /* end _xmit_reply() */ + +/* --------------------------------------------------------------------------- + * Function: reply_ok + * + * Purpose: Send an OK reply through the session. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +reply_ok(struct mirror_session *session) +{ + H5FD_mirror_xmit_reply_t *reply = &(session->reply); + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_ALL, "reply_ok()"); + + reply->status = H5FD_MIRROR_STATUS_OK; + mybzero(reply->message, H5FD_MIRROR_STATUS_MESSAGE_MAX); + return _xmit_reply(session); +} /* end reply_ok() */ + +/* --------------------------------------------------------------------------- + * Function: reply_error + * + * Purpose: Send an ERROR reply with message through the session. + * Message may be cut short if it would overflow the available + * buffer in the xmit. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +reply_error(struct mirror_session *session, const char *msg) +{ + H5FD_mirror_xmit_reply_t *reply = &(session->reply); + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_ALL, "reply_error(%s)", msg); + + reply->status = H5FD_MIRROR_STATUS_ERROR; + HDsnprintf(reply->message, H5FD_MIRROR_STATUS_MESSAGE_MAX - 1, "%s", msg); + return _xmit_reply(session); +} /* end reply_error() */ + +/* --------------------------------------------------------------------------- + * Function: do_close + * + * Purpose: Handle an CLOSE operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_close(struct mirror_session *session) +{ + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "do_close()"); + + if (NULL == session->file) { + mirror_log(session->loginfo, V_ERR, "no file to close!"); + reply_error(session, "no file to close"); + return -1; + } + + if (H5FDclose(session->file) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDclose()"); + reply_error(session, "H5FDclose()"); + return -1; + } + session->file = NULL; + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_close() */ + +/* --------------------------------------------------------------------------- + * Function: do_lock + * + * Purpose: Handle a LOCK operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_lock(struct mirror_session *session, const unsigned char *xmit_buf) +{ + size_t decode_ret = 0; + H5FD_mirror_xmit_lock_t xmit_lock; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_buf); + + mirror_log(session->loginfo, V_INFO, "do_lock()"); + + decode_ret = H5FD_mirror_xmit_decode_lock(&xmit_lock, xmit_buf); + if (H5FD_MIRROR_XMIT_LOCK_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, "can't decode set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decoding size failure"); + return -1; + } + + if (!H5FD_mirror_xmit_is_lock(&xmit_lock)) { + mirror_log(session->loginfo, V_ERR, "not a set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decode failure"); + return -1; + } + mirror_log(session->loginfo, V_INFO, "lock rw: (%d)", xmit_lock.rw); + + if (H5FDlock(session->file, (hbool_t)xmit_lock.rw) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDlock()"); + reply_error(session, "remote H5FDlock() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_lock() */ + +/* --------------------------------------------------------------------------- + * Function: do_open + * + * Purpose: Handle an OPEN operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_open(struct mirror_session *session, const H5FD_mirror_xmit_open_t *xmit_open) +{ + hid_t fapl_id = H5I_INVALID_HID; + unsigned _flags = 0; + haddr_t _maxaddr = HADDR_UNDEF; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_open && + TRUE == H5FD_mirror_xmit_is_open(xmit_open)); + + mirror_log(session->loginfo, V_INFO, "do_open()"); + + if (0 != xmit_open->pub.xmit_count) { + mirror_log(session->loginfo, V_ERR, "open with xmit count not zero!"); + reply_error(session, "initial transmission count not zero"); + goto error; + } + if (0 != session->token) { + mirror_log(session->loginfo, V_ERR, "open with token already set!"); + reply_error(session, "initial session token not zero"); + goto error; + } + + session->xmit_count = 1; + session->token = xmit_open->pub.session_token; + session->reply.pub.session_token = session->token; + + _flags = (unsigned)xmit_open->flags; + _maxaddr = (haddr_t)xmit_open->maxaddr; + + /* Check whether the native size_t on the remote machine (Driver) is larger + * than that on the local machine; if so, issue a warning. + * The blob is always an 8-byte bitfield -- check its contents. + */ + if (xmit_open->size_t_blob > (uint64_t)((size_t)(-1))) { + mirror_log(session->loginfo, V_WARN, "Driver size_t is larger than our own"); + } + + mirror_log(session->loginfo, V_INFO, "to open file %s (flags %d) (maxaddr %d)", xmit_open->filename, + _flags, _maxaddr); + + /* Explicitly use Sec2 as the underlying driver for now. + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (fapl_id < 0) { + mirror_log(session->loginfo, V_ERR, "can't create FAPL"); + reply_error(session, "H5Pcreate() failure"); + goto error; + } + if (H5Pset_fapl_sec2(fapl_id) < 0) { + mirror_log(session->loginfo, V_ERR, "can't set FAPL as Sec2"); + reply_error(session, "H5Pset_fapl_sec2() failure"); + goto error; + } + + session->file = H5FDopen(xmit_open->filename, _flags, fapl_id, _maxaddr); + if (NULL == session->file) { + mirror_log(session->loginfo, V_ERR, "H5FDopen()"); + reply_error(session, "remote H5FDopen() failure"); + goto error; + } + + /* FAPL is set and in use; clean up */ + if (H5Pclose(fapl_id) < 0) { + mirror_log(session->loginfo, V_ERR, "can't set FAPL as Sec2"); + reply_error(session, "H5Pset_fapl_sec2() failure"); + goto error; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; + +error: + if (fapl_id > 0) { + H5E_BEGIN_TRY + { + (void)H5Pclose(fapl_id); + } + H5E_END_TRY; + } + return -1; +} /* end do_open() */ + +/* --------------------------------------------------------------------------- + * Function: do_set_eoa + * + * Purpose: Handle a SET_EOA operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_set_eoa(struct mirror_session *session, const unsigned char *xmit_buf) +{ + size_t decode_ret = 0; + H5FD_mirror_xmit_eoa_t xmit_seoa; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_buf); + + mirror_log(session->loginfo, V_INFO, "do_set_eoa()"); + + decode_ret = H5FD_mirror_xmit_decode_set_eoa(&xmit_seoa, xmit_buf); + if (H5FD_MIRROR_XMIT_EOA_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, "can't decode set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decoding size failure"); + return -1; + } + + if (!H5FD_mirror_xmit_is_set_eoa(&xmit_seoa)) { + mirror_log(session->loginfo, V_ERR, "not a set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decode failure"); + return -1; + } + + mirror_log(session->loginfo, V_INFO, "set EOA addr %d", xmit_seoa.eoa_addr); + + if (H5FDset_eoa(session->file, (H5FD_mem_t)xmit_seoa.type, (haddr_t)xmit_seoa.eoa_addr) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDset_eoa()"); + reply_error(session, "remote H5FDset_eoa() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_set_eoa() */ + +/* --------------------------------------------------------------------------- + * Function: do_truncate + * + * Purpose: Handle a TRUNCATE operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_truncate(struct mirror_session *session) +{ + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "do_truncate()"); + + /* default DXPL ID (0), 0 for "FALSE" closing -- both probably unused */ + if (H5FDtruncate(session->file, 0, 0) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDtruncate()"); + reply_error(session, "remote H5FDtruncate() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_truncate() */ + +/* --------------------------------------------------------------------------- + * Function: do_unlock + * + * Purpose: Handle an UNLOCK operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_unlock(struct mirror_session *session) +{ + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "do_unlock()"); + + if (H5FDunlock(session->file) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDunlock()"); + reply_error(session, "remote H5FDunlock() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_unlock() */ + +/* --------------------------------------------------------------------------- + * Function: do_write + * + * Purpose: Handle a WRITE operation. + * Receives command, replies; receives & writes data, replies. + * + * It is known that this results in suboptimal performance, + * but handling both small and very, very large write buffers + * with a single "over the wire" exchange + * poses design challenges not worth tackling as of March 2020. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_write(struct mirror_session *session, const unsigned char *xmit_buf) +{ + size_t decode_ret = 0; + haddr_t addr = 0; + haddr_t sum_bytes_written = 0; + H5FD_mem_t type = 0; + char * buf = NULL; + ssize_t nbytes_in_packet = 0; + H5FD_mirror_xmit_write_t xmit_write; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_buf); + + mirror_log(session->loginfo, V_INFO, "do_write()"); + + if (NULL == session->file) { + mirror_log(session->loginfo, V_ERR, "no open file!"); + reply_error(session, "no file open on remote"); + return -1; + } + + decode_ret = H5FD_mirror_xmit_decode_write(&xmit_write, xmit_buf); + if (H5FD_MIRROR_XMIT_WRITE_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, "can't decode write xmit"); + reply_error(session, "remote xmit_write_t decoding size failure"); + return -1; + } + + if (!H5FD_mirror_xmit_is_write(&xmit_write)) { + mirror_log(session->loginfo, V_ERR, "not a write xmit"); + reply_error(session, "remote xmit_write_t decode failure"); + return -1; + } + + addr = (haddr_t)xmit_write.offset; + type = (H5FD_mem_t)xmit_write.type; + + /* Allocate the buffer once -- re-use between loops. + */ + buf = (char *)HDmalloc(sizeof(char) * H5FD_MIRROR_DATA_BUFFER_MAX); + if (NULL == buf) { + mirror_log(session->loginfo, V_ERR, "can't allocate databuffer"); + reply_error(session, "can't allocate buffer for receiving data"); + return -1; + } + + /* got write signal; ready for data */ + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + mirror_log(session->loginfo, V_INFO, "to write %zu bytes at %zu", xmit_write.size, addr); + + /* The given write may be: + * 1. larger than the allowed single buffer size + * 2. larger than the native size_t of this system + * + * Handle all cases by looping, ingesting as much of the stream as possible + * and writing that part to the file. + */ + sum_bytes_written = 0; + do { + if ((nbytes_in_packet = HDread(session->sockfd, buf, H5FD_MIRROR_DATA_BUFFER_MAX)) < 0) { + mirror_log(session->loginfo, V_ERR, "can't read into databuffer"); + reply_error(session, "can't read data buffer"); + return -1; + } + + mirror_log(session->loginfo, V_INFO, "received %zd bytes", nbytes_in_packet); + if (HEXDUMP_WRITEDATA) { + mirror_log(session->loginfo, V_ALL, "DATA:\n```"); + mirror_log_bytes(session->loginfo, V_ALL, (size_t)nbytes_in_packet, (const unsigned char *)buf); + mirror_log(session->loginfo, V_ALL, "```"); + } + + mirror_log(session->loginfo, V_INFO, "writing %zd bytes at %zu", nbytes_in_packet, + (addr + sum_bytes_written)); + + if (H5FDwrite(session->file, type, H5P_DEFAULT, (addr + sum_bytes_written), (size_t)nbytes_in_packet, + buf) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDwrite()"); + reply_error(session, "remote H5FDwrite() failure"); + return -1; + } + + sum_bytes_written += (haddr_t)nbytes_in_packet; + + } while (sum_bytes_written < xmit_write.size); /* end while ingesting */ + + HDfree(buf); + + /* signal that we're done here and a-ok */ + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_write() */ + +/* --------------------------------------------------------------------------- + * Function: receive_communique + * + * Purpose: Accept bytes from the socket, check for emergency shutdown, and + * sanity-check received bytes. + * The raw bytes read are stored in the sock_comm structure at + * comm->raw. + * The raw bytes are decoded and a xmit_t (header) struct pointer + * in comm is populated at comm->xmit_recd. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +receive_communique(struct mirror_session *session, struct sock_comm *comm) +{ + ssize_t read_ret = 0; + size_t decode_ret; + H5FD_mirror_xmit_t *X = comm->xmit_recd; + + HDassert((session != NULL) && (session->magic == MW_SESSION_MAGIC) && (comm != NULL) && + (comm->magic == MW_SOCK_COMM_MAGIC) && (comm->xmit_recd != NULL) && (comm->raw != NULL) && + (comm->raw_size >= H5FD_MIRROR_XMIT_BUFFER_MAX)); + + mirror_log(session->loginfo, V_INFO, "receive_communique()"); + + mybzero(comm->raw, comm->raw_size); + comm->recd_die = 0; + + mirror_log(session->loginfo, V_INFO, "ready to receive"); /* TODO */ + + if ((read_ret = HDread(session->sockfd, comm->raw, H5FD_MIRROR_XMIT_BUFFER_MAX)) < 0) { + mirror_log(session->loginfo, V_ERR, "read:%zd", read_ret); + goto error; + } + + mirror_log(session->loginfo, V_INFO, "received %zd bytes", read_ret); + if (HEXDUMP_XMITS) { + mirror_log(session->loginfo, V_ALL, "```", read_ret); + mirror_log_bytes(session->loginfo, V_ALL, (size_t)read_ret, (const unsigned char *)comm->raw); + mirror_log(session->loginfo, V_ALL, "```"); + } /* end if hexdump transmissions received */ + + /* old-fashioned manual kill (for debugging) */ + if (!HDstrncmp("GOODBYE", comm->raw, 7)) { + mirror_log(session->loginfo, V_INFO, "received GOODBYE"); + comm->recd_die = 1; + goto done; + } + + decode_ret = H5FD_mirror_xmit_decode_header(X, (const unsigned char *)comm->raw); + if (H5FD_MIRROR_XMIT_HEADER_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, "header decode size mismatch: expected (%z), got (%z)", + H5FD_MIRROR_XMIT_HEADER_SIZE, decode_ret); + /* Try to tell Driver that it should stop */ + reply_error(session, "xmit size mismatch"); + goto error; + } + + if (!H5FD_mirror_xmit_is_xmit(X)) { + mirror_log(session->loginfo, V_ERR, "bad magic: 0x%X", X->magic); + /* Try to tell Driver that it should stop */ + reply_error(session, "bad magic"); + goto error; + } + + if (session->xmit_count != X->xmit_count) { + mirror_log(session->loginfo, V_ERR, "xmit_count mismatch exp:%d recd:%d", session->xmit_count, + X->xmit_count); + /* Try to tell Driver that it should stop */ + reply_error(session, "xmit_count mismatch"); + goto error; + } + + if ((session->token > 0) && (session->token != X->session_token)) { + mirror_log(session->loginfo, V_ERR, "wrong session"); + /* Try to tell Driver that it should stop */ + reply_error(session, "wrong session"); + goto error; + } + + session->xmit_count++; + +done: + return 0; + +error: + return -1; +} /* end receive_communique() */ + +/* --------------------------------------------------------------------------- + * Function: process_instructions + * + * Purpose: Receive and handle all instructions from Driver. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +process_instructions(struct mirror_session *session) +{ + struct sock_comm comm; + char xmit_buf[H5FD_MIRROR_XMIT_BUFFER_MAX]; /* raw bytes */ + H5FD_mirror_xmit_t xmit_recd; /* for decoded xmit header */ + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "process_instructions()"); + + comm.magic = MW_SOCK_COMM_MAGIC; + comm.recd_die = 0; /* Flag for program to terminate */ + comm.xmit_recd = &xmit_recd; + comm.raw = xmit_buf; + comm.raw_size = sizeof(xmit_buf); + + while (1) { /* sill-listening infinite loop */ + + /* Use convenience structure for raw/decoded info in/out */ + if (receive_communique(session, &comm) < 0) { + mirror_log(session->loginfo, V_ERR, "problem reading socket"); + return -1; + } + + if (comm.recd_die) { + goto done; + } + + switch (xmit_recd.op) { + case H5FD_MIRROR_OP_CLOSE: + if (do_close(session) < 0) { + return -1; + } + goto done; + case H5FD_MIRROR_OP_LOCK: + if (do_lock(session, (const unsigned char *)xmit_buf) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_OPEN: + mirror_log(session->loginfo, V_ERR, "OPEN xmit during session"); + reply_error(session, "illegal OPEN xmit during session"); + return -1; + case H5FD_MIRROR_OP_SET_EOA: + if (do_set_eoa(session, (const unsigned char *)xmit_buf) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_TRUNCATE: + if (do_truncate(session) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_UNLOCK: + if (do_unlock(session) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_WRITE: + if (do_write(session, (const unsigned char *)xmit_buf) < 0) { + return -1; + } + break; + default: + mirror_log(session->loginfo, V_ERR, "unrecognized transmission"); + reply_error(session, "unrecognized transmission"); + return -1; + } /* end switch (xmit_recd.op) */ + + } /* end while still listening */ + +done: + comm.magic = 0; /* invalidate structure, on principle */ + xmit_recd.magic = 0; /* invalidate structure, on principle */ + return 0; +} /* end process_instructions() */ + +/* --------------------------------------------------------------------------- + * Function: run_writer + * + * Purpose: Initiate Writer operations. + * + * Receives as parameters a socket which has accepted the + * connection to the Driver and the OPEN xmit (which must be + * decoded into the structure and verified prior to being passed + * to this function). + * + * Is not responsible for closing or cleaning up any of the + * received parameters. + * + * Return: Success: SUCCEED + * Failure: FAIL + * --------------------------------------------------------------------------- + */ +herr_t +run_writer(int socketfd, H5FD_mirror_xmit_open_t *xmit_open) +{ + struct mirror_session *session = NULL; + int ret_value = SUCCEED; + + session = session_start(socketfd, xmit_open); + if (NULL == session) { + mirror_log(NULL, V_ERR, "Can't start session -- aborting"); + ret_value = FAIL; + } + else { + if (process_instructions(session) < 0) { + mirror_log(session->loginfo, V_ERR, "problem processing instructions"); + ret_value = FAIL; + } + if (session_stop(session) < 0) { + mirror_log(NULL, V_ERR, "Can't stop session -- going down hard"); + ret_value = FAIL; + } + } + + return ret_value; +} /* end run_writer */ + +#endif /* H5_HAVE_MIRROR_VFD */ diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt new file mode 100644 index 0000000..921fbd0 --- /dev/null +++ b/utils/test/CMakeLists.txt @@ -0,0 +1,35 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TEST C) + +################################################################################# +# Test program sources +################################################################################# + +macro (ADD_H5_EXE file) + add_executable (${file} ${HDF5_TEST_SOURCE_DIR}/${file}.c) + target_include_directories (${file} PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TEST_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(${file} PRIVATE "${HDF5_CMAKE_C_FLAGS}") + if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (${file} STATIC) + target_link_libraries (${file} PRIVATE ${HDF5_TEST_LIB_TARGET}) + else () + TARGET_C_PROPERTIES (${file} SHARED) + target_link_libraries (${file} PRIVATE ${HDF5_TEST_LIBSH_TARGET}) + endif () + set_target_properties (${file} PROPERTIES FOLDER test) +endmacro () + +############################################################################## +### S W I M M E R T E S T U T I L S ### +############################################################################## +set (H5_UTIL_TESTS) + +if (HDF5_TEST_SWMR) + set (H5_UTIL_TESTS ${H5_UTIL_TESTS} swmr_check_compat_vfd) +endif () + +if (H5_UTIL_TESTS) + foreach (h5_test ${H5_UTIL_TESTS}) + ADD_H5_EXE(${h5_test}) + endforeach () +endif () diff --git a/utils/test/Makefile.am b/utils/test/Makefile.am new file mode 100644 index 0000000..164562f --- /dev/null +++ b/utils/test/Makefile.am @@ -0,0 +1,34 @@ +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src and tools/lib directories +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -I$(top_srcdir)/utils/test + +# These are our main targets, the tools + +noinst_PROGRAMS=swmr_check_compat_vfd + +# Programs all depend on the hdf5 library, the tools library, and the HL +# library. +LDADD=$(LIBH5TEST) $(LIBHDF5) + +CHECK_CLEANFILES+=*.h5 + +include $(top_srcdir)/config/conclude.am diff --git a/utils/test/swmr_check_compat_vfd.c b/utils/test/swmr_check_compat_vfd.c new file mode 100644 index 0000000..720c747 --- /dev/null +++ b/utils/test/swmr_check_compat_vfd.c @@ -0,0 +1,53 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Purpose: This is a small program that checks if the HDF5_DRIVER + * environment variable is set to a value that supports SWMR. + * + * It is intended for use in shell scripts. + */ + +#include "h5test.h" + +/* This file needs to access the file driver testing code */ +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ +#define H5FD_TESTING +#include "H5FDpkg.h" /* File drivers */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Inspects the HDF5_DRIVER environment variable, which + * determines the VFD that the test harness will use with + * the majority of the tests. + * + * Return: VFD supports SWMR: EXIT_SUCCESS + * + * VFD does not support SWMR + * or failure: EXIT_FAILURE + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + char *driver = NULL; + + driver = HDgetenv(HDF5_DRIVER); + + if (H5FD__supports_swmr_test(driver)) + return EXIT_SUCCESS; + else + return EXIT_FAILURE; + +} /* end main() */ diff --git a/utils/tools/CMakeLists.txt b/utils/tools/CMakeLists.txt new file mode 100644 index 0000000..49562d7 --- /dev/null +++ b/utils/tools/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_UTILS_TOOLS C) + + +if (HDF5_BUILD_PARALLEL_TOOLS) + add_subdirectory (h5dwalk) +endif() + +#-- Add the tests +if (BUILD_TESTING) + add_subdirectory (test) +endif() diff --git a/utils/tools/Makefile.am b/utils/tools/Makefile.am new file mode 100644 index 0000000..0c89aff --- /dev/null +++ b/utils/tools/Makefile.am @@ -0,0 +1,38 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Tools HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +if PARALLEL_TOOLS_CONDITIONAL + H5DWALK=h5dwalk +else + H5DWALK= +endif + +if BUILD_TESTS_CONDITIONAL + TESTSERIAL_DIR =test +else + TESTSERIAL_DIR= +endif + +CONFIG=ordered + +# All subdirectories +SUBDIRS=$(H5DWALK) $(TESTSERIAL_DIR) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/h5dwalk/CMakeLists.txt b/utils/tools/h5dwalk/CMakeLists.txt new file mode 100644 index 0000000..244cc26 --- /dev/null +++ b/utils/tools/h5dwalk/CMakeLists.txt @@ -0,0 +1,66 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_UTILS_TOOLS_H5DWALK C) + +# -------------------------------------------------------------------- +# Add the h5dwalk and test executables +# -------------------------------------------------------------------- +if (NOT ONLY_SHARED_LIBS) + add_executable (h5dwalk ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) +# add_custom_target(generate_demo ALL +# DEPENDS "${HDF5_TOOLS_DIR}/test/demo_destfiles.test" +# ) + target_include_directories (h5dwalk PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(h5dwalk PRIVATE "${HDF5_CMAKE_C_FLAGS}") + TARGET_C_PROPERTIES (h5dwalk STATIC) + target_link_libraries (h5dwalk PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${MFU_LIBRARY} "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>") + set_target_properties (h5dwalk PROPERTIES FOLDER tools) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk") + + set (H5_DEP_EXECUTABLES h5dwalk) +endif () + +if (BUILD_SHARED_LIBS) + add_executable (h5dwalk-shared ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) + target_include_directories (h5dwalk-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(h5dwalk-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") + TARGET_C_PROPERTIES (h5dwalk-shared SHARED) + target_link_libraries (h5dwalk-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${MFU_LIBRARY} "$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_LIBRARIES}>") + set_target_properties (h5dwalk-shared PROPERTIES FOLDER tools) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk-shared") + + set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} h5dwalk-shared) +endif () + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + if (NOT ONLY_SHARED_LIBS) + clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk) + else () + clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk-shared) + endif () +endif () + +############################################################################## +############################################################################## +### I N S T A L L A T I O N ### +############################################################################## +############################################################################## + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- +if (HDF5_EXPORTED_TARGETS) + foreach (exec ${H5_DEP_EXECUTABLES}) + INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} toolsapplications) + endforeach () + + install ( + TARGETS + ${H5_DEP_EXECUTABLES} + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications + ) +endif () diff --git a/utils/tools/h5dwalk/Makefile.am b/utils/tools/h5dwalk/Makefile.am new file mode 100644 index 0000000..34cdb32 --- /dev/null +++ b/utils/tools/h5dwalk/Makefile.am @@ -0,0 +1,37 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src directory +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib $(H5DWALK_CPPFLAGS) + +# These are our main targets, the tools +# h5dwalk_SOURCES=h5dwalk.c $(TOOLSOURCES) +bin_PROGRAMS=h5dwalk +#bin_SCRIPTS=install-examples + +# Add h5stat specific linker flags here +h5dwalk_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) $(H5DWALK_LDFLAGS) + +# Tell automake to clean h5redeploy script +CLEANFILES= + +# All programs rely on hdf5 library and h5tools library +h5dwalk_LDADD=$(LIBH5TOOLS) $(LIBHDF5) $(H5DWALK_LIBS) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/h5dwalk/h5dwalk.1 b/utils/tools/h5dwalk/h5dwalk.1 new file mode 100644 index 0000000..eb0e5e8 --- /dev/null +++ b/utils/tools/h5dwalk/h5dwalk.1 @@ -0,0 +1,42 @@ +.TH "h5dwalk" 1 +.SH NAME +h5dwalk \- Provides a means of extending HDF5 tools by using parallelism on groups of files. +.SH SYNOPSIS +h5dwalk [OPTIONS] -T h5tool [H5TOOL_options...] +.SH DESCRIPTION +h5dwalk utilizes the mpiFileUtils library to invoke a selected HDF5 tool on a collection of files. The mpiFileUtils library provides the facilities to walk directory trees and provide a selection of files contained therein. This selection can be filtered in various ways. At present, h5dwalk filters the original file selection to include only HDF5 formatted files. The resulting collection or collections can be utilized as the file inputs to the selected h5tool. +.SH OPTIONS +.TP +.B \-h +or +.B \-\-help +Print a usage message and exit. +.TP +.B \-i +or +.B \-\-input filename +Read command input from a file. Not yet implemented. +.TP +.B \-o +or +.B \-\-output filename +Captures the hdf5 tool output into a named file. +.TP +.B \-l +or +.B \-\-log [file] +Captures hdf5 tool output into a individual log files. If an optional file (directory) is specified, then output from all tool instances will be written in the given file directory. Without the optional filename, each tool instance output will be captured in a new log file whose name is associated with the hdf5 tool that was run and is written in the current working directory. +.TP +.B \-E +or +.B \-\-error [file] +Show all HDF5 error reporting. Behavior is similar to --log, i.e. errors can either be logged in a single named file or in individual tool specific files. Not yet implemented. +.TP +.B \-T +or +.B \-\-tool hdf5_tool +Specifies the hdf5 tool that should be invoked for each file in a collection of files. The collection consists of individual HDF5 files found by walking a specified directory tree which is used in place of the normal tool filename argument. The '-T' option should appear on the command line just prior to the HDF5 tool argument options. +.TP +.SH "SEE ALSO" +\&\fIh5dump\fR\|(1), \fIh5diff\fR\|(1), \fIh5repart\fR\|(1), \fIh5diff\fR\|(1), +\&\fIh5import\fR\|(1), \fIgif2h5\fR\|(1), \fIh52gif\fR\|(1), \fIh5perf\fR\|(1) diff --git a/utils/tools/h5dwalk/h5dwalk.c b/utils/tools/h5dwalk/h5dwalk.c new file mode 100644 index 0000000..5a22d75 --- /dev/null +++ b/utils/tools/h5dwalk/h5dwalk.c @@ -0,0 +1,1714 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "H5private.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#include "hdf5.h" + +#include "libcircle.h" +#include "dtcmp.h" +#include "mfu.h" +#include "mfu_flist.h" +#include "mfu_errors.h" +#include "mfu_flist_internal.h" + +/* Name of tool */ +#define PROGRAMNAME "h5dwalk" + +#ifdef DAOS_SUPPORT +#include "mfu_daos.h" +#endif + +static char *user_cmd = NULL; +static char mpierrstr[MPI_MAX_ERROR_STRING]; +static int mpierrlen; +static int sg_mpi_rank = 0; +static int current_input_index = 0; +static int processing_inputfile = 0; + +static void dh5tool_flist_write_text(const char *name, mfu_flist bflist); +static void run_command(int argc, char **argv, char *cmdline, const char *fname); +static void add_executable(int argc, char **argv, char *cmdstring, int *f_index, int f_count); +static int process_input_file(char *inputname, int myrank, int size); +static void usage(void); + +H5_ATTR_NORETURN void h5dwalk_exit(int status); + +/* keep stats during walk */ +uint64_t total_dirs = 0; +uint64_t total_files = 0; +uint64_t total_links = 0; +uint64_t total_unknown = 0; +uint64_t total_bytes = 0; +/* global flags which indicate whether we need + * to capture tool outputs into a file... + * Related to this is whether the stderr should + * be logged separately. + */ +#define BUFT_SIZE 131072 +/* FIXME: 'buft_max' should probably be configurable.. */ +size_t buft_max = 64; +size_t buft_count = 0; +buf_t **buf_cache = NULL; + +int log_output_in_single_file = 0; +char *output_log_file = NULL; + +int log_stdout_in_file = 0; +char *txtlog = NULL; + +int log_errors_in_file = 0; +char *errlog = NULL; + +int use_config_file = 0; +int config_index[4] = { + 0, +}; + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +#define MAX_DISTRIBUTE_SEPARATORS 128 +struct distribute_option { + int separator_number; + uint64_t separators[MAX_DISTRIBUTE_SEPARATORS]; +}; + +static const char * s_opts = "hl*E*i:o:T:"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, + {"log_text", optional_arg, 'l'}, + {"error", optional_arg, 'E'}, + {"input", require_arg, 'i'}, + {"output", require_arg, 'o'}, + {"tool", require_arg, 'T'}, + {NULL, 0, '\0'}}; +static void +save_command(const char *argv0) +{ + assert(argv0); + user_cmd = HDstrdup(argv0); +} + +static void +create_default_separators(struct distribute_option *option, mfu_flist *flist, uint64_t *size, + size_t *separators, uint64_t *global_max_file_size) +{ + /* get local max file size for Allreduce */ + uint64_t local_max_file_size = 0; + for (uint64_t i = 0; i < *size; i++) { + uint64_t file_size = mfu_flist_file_get_size(*flist, i); + if (file_size > local_max_file_size) { + local_max_file_size = file_size; + } + } + + /* get the max file size across all ranks */ + MPI_Allreduce(&local_max_file_size, global_max_file_size, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); + + /* print and convert max file size to appropriate units */ + double max_size_tmp; + const char *max_size_units; + mfu_format_bytes(*global_max_file_size, &max_size_tmp, &max_size_units); + HDprintf("Max File Size: %.3lf %s\n", max_size_tmp, max_size_units); + + /* round next_pow_2 to next multiple of 10 */ + uint64_t max_magnitude_bin = (uint64_t)((ceil(log2((double)(*global_max_file_size)) / 10)) * 10); + + /* get bin ranges based on max file size */ + option->separators[0] = 1; + + /* plus one is for zero count bin */ + *separators = (size_t)(max_magnitude_bin / 10); + uint64_t power = 10; + for (int i = 1; power <= max_magnitude_bin; i++) { + double raised_2 = pow(2, (double)(power)); + option->separators[i] = (uint64_t)raised_2; + power += 10; + } +} + +static int +h5dwalk_map_fn(mfu_flist flist __attribute__((unused)), uint64_t idx, int ranks, + void *args __attribute__((unused))) +{ + int rank = (int)((int)idx % ranks); + return rank; +} + +static int +print_flist_distribution(int file_histogram, struct distribute_option *option, mfu_flist *pflist, int rank) +{ + /* file list to use */ + mfu_flist flist = *pflist; + + /* get local size for each rank, and max file sizes */ + uint64_t size = mfu_flist_size(flist); + uint64_t global_max_file_size; + + size_t separators = 0; + if (file_histogram) { + /* create default separators */ + create_default_separators(option, &flist, &size, &separators, &global_max_file_size); + } + else { + separators = (size_t)option->separator_number; + } + + /* allocate a count for each bin, initialize the bin counts to 0 + * it is separator + 1 because the last bin is the last separator + * to the DISTRIBUTE_MAX */ + uint64_t *dist = (uint64_t *)MFU_MALLOC((separators + 1) * sizeof(uint64_t)); + + /* initialize the bin counts to 0 */ + for (size_t i = 0; i <= separators; i++) { + dist[i] = 0; + } + + /* for each file, identify appropriate bin and increment its count */ + for (size_t i = 0; i < size; i++) { + /* get the size of the file */ + uint64_t file_size = mfu_flist_file_get_size(flist, i); + + /* loop through the bins and find the one the file belongs to, + * set last bin to -1, if a bin is not found while looping through the + * list of file size separators, then it belongs in the last bin + * so (last file size - MAX bin) */ + int64_t max_bin_flag = -1; + for (size_t j = 0; j < separators; j++) { + if (file_size <= option->separators[j]) { + /* found the bin set bin index & increment its count */ + dist[j]++; + + /* a file for this bin was found so can't belong to + * last bin (so set the flag) & exit the loop */ + max_bin_flag = 1; + break; + } + } + + /* if max_bin_flag is still -1 then the file belongs to the last bin */ + if (max_bin_flag < 0) { + dist[separators]++; + } + } + + /* get the total sum across all of the bins */ + uint64_t *disttotal = (uint64_t *)MFU_MALLOC((separators + 1) * sizeof(uint64_t)); + MPI_Allreduce(dist, disttotal, (int)(separators + 1), MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); + + /* Print the file distribution */ + if (rank == 0) { + /* number of files in a bin */ + uint64_t number; + double size_tmp; + const char *size_units; + HDprintf("%-27s %s\n", "Range", "Number"); + for (size_t i = 0; i <= separators; i++) { + HDprintf("%s", "[ "); + if (i == 0) { + HDprintf("%7.3lf %3s", 0.000, "B"); + } + else { + mfu_format_bytes((uint64_t)option->separators[i - 1], &size_tmp, &size_units); + HDprintf("%7.3lf %3s", size_tmp, size_units); + } + + printf("%s", " - "); + + if (file_histogram) { + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + number = disttotal[i]; + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + HDprintf("%7.3lf %3s ) %" PRIu64 "\n", size_tmp, size_units, number); + } + else { + if (i == separators) { + number = disttotal[i]; + HDprintf("%10s ) %" PRIu64 "\n", "MAX", number); + } + else { + number = disttotal[i]; + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + HDprintf("%7.3lf %3s ) %" PRIu64 "\n", size_tmp, size_units, number); + } + } + } + } + + /* free the memory used to hold bin counts */ + mfu_free(&disttotal); + mfu_free(&dist); + + return 0; +} + +/* * Search the right position to insert the separator * If the separator exists already, return failure * + * Otherwise, locate the right position, and move the array forward to save the separator. + */ +static int +distribute_separator_add(struct distribute_option *option, uint64_t separator) +{ + int low = 0; + int high; + int middle; + int pos; + int count; + + count = option->separator_number; + option->separator_number++; + if (option->separator_number > MAX_DISTRIBUTE_SEPARATORS) { + HDprintf("Too many separators"); + return -1; + } + + if (count == 0) { + option->separators[0] = separator; + return 0; + } + + high = count - 1; + while (low < high) { + middle = (high - low) / 2 + low; + if (option->separators[middle] == separator) + return -1; + /* In the left half */ + else if (option->separators[middle] < separator) + low = middle + 1; + /* In the right half */ + else + high = middle; + } + assert(low == high); + if (option->separators[low] == separator) + return -1; + + if (option->separators[low] < separator) + pos = low + 1; + else + pos = low; + + if (pos < count) + HDmemmove(&option->separators[low + 1], &option->separators[low], + sizeof(*option->separators) * (uint64_t)(count - pos)); + + option->separators[pos] = separator; + return 0; +} + +static int +distribution_parse(struct distribute_option *option, const char *string) +{ + char * ptr; + char * next; + unsigned long long separator; + char * str; + int status = 0; + + if (strncmp(string, "size", strlen("size")) != 0) { + return -1; + } + + option->separator_number = 0; + if (strlen(string) == strlen("size")) { + return 0; + } + + if (string[strlen("size")] != ':') { + return -1; + } + + str = HDstrdup(string); + /* Parse separators */ + ptr = str + strlen("size:"); + next = ptr; + while (ptr && ptr < str + strlen(string)) { + next = strchr(ptr, ','); + if (next != NULL) { + *next = '\0'; + next++; + } + + if (mfu_abtoull(ptr, &separator) != MFU_SUCCESS) { + HDprintf("Invalid separator \"%s\"\n", ptr); + status = -1; + goto out; + } + + if (distribute_separator_add(option, separator)) { + HDprintf("Duplicated separator \"%llu\"\n", separator); + status = -1; + goto out; + } + + ptr = next; + } + +out: + mfu_free(&str); + return status; +} + +static void +usage(void) +{ + if (sg_mpi_rank) + return; + + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "Usage: h5dwalk [options] <path> ...\n"); +#ifdef DAOS_SUPPORT + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "DAOS paths can be specified as:\n"); + PRINTVALSTREAM(rawoutstream, " daos://<pool>/<cont>[/<path>] | <UNS path>\n"); +#endif + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "Options:\n"); + PRINTVALSTREAM(rawoutstream, " -i, --input <file> - read list from file\n"); + PRINTVALSTREAM(rawoutstream, " -o, --output <file> - write output summary to the named file.\n"); + PRINTVALSTREAM(rawoutstream, + " -E, --error <file> - write processed errors to file in text format\n"); + PRINTVALSTREAM( + rawoutstream, + " -l, --log_text <dir> - write individual tool outputs to a file. Logs can be written to an " + "optional named directory.\n"); + PRINTVALSTREAM(rawoutstream, " -T, --tool <executable> - name of the HDF5 tool to invoke\n"); + PRINTVALSTREAM(rawoutstream, " -h, --help - print usage\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "For more information see https://mpifileutils.readthedocs.io. \n"); + PRINTVALSTREAM(rawoutstream, "\n"); +} + +/* given an index, return pointer to that file element, + * NULL if index is not in range */ +static elem_t * +list_get_elem(flist_t *flist, uint64_t idx) +{ + /* return pointer to element if index is within range */ + uint64_t max = flist->list_count; + if (idx < max) { + elem_t *elem = flist->list_index[idx]; + return elem; + } + return NULL; +} + +#ifdef VERBOSE +/* print information about a file given the index and rank (used in print_files) */ +static void +print_file(mfu_flist flist, uint64_t idx) +{ + /* store types as strings for print_file */ + char type_str_unknown[] = "UNK"; + char type_str_dir[] = "DIR"; + char type_str_file[] = "REG"; + char type_str_link[] = "LNK"; + + /* get filename */ + const char *file = mfu_flist_file_get_name(flist, idx); + + if (mfu_flist_have_detail(flist)) { + /* get mode */ + mode_t mode = (mode_t)mfu_flist_file_get_mode(flist, idx); + uint64_t acc = mfu_flist_file_get_atime(flist, idx); + uint64_t mod = mfu_flist_file_get_mtime(flist, idx); + uint64_t cre = mfu_flist_file_get_ctime(flist, idx); + uint64_t size = mfu_flist_file_get_size(flist, idx); + const char *username = mfu_flist_file_get_username(flist, idx); + const char *groupname = mfu_flist_file_get_groupname(flist, idx); + + char access_s[30]; + char modify_s[30]; + char create_s[30]; + time_t access_t = (time_t)acc; + time_t modify_t = (time_t)mod; + time_t create_t = (time_t)cre; + size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t)); + size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t)); + size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t)); + if (access_rc == 0 || modify_rc == 0 || create_rc == 0) { + /* error */ + access_s[0] = '\0'; + modify_s[0] = '\0'; + create_s[0] = '\0'; + } + + char mode_format[11]; + mfu_format_mode(mode, mode_format); + + double size_tmp; + const char *size_units; + mfu_format_bytes(size, &size_tmp, &size_units); + + HDprintf("%s %s %s %7.3f %3s %s %s\n", mode_format, username, groupname, size_tmp, size_units, + modify_s, file); + } + else { + /* get type */ + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + char * type_str = type_str_unknown; + if (type == MFU_TYPE_DIR) { + type_str = type_str_dir; + } + else if (type == MFU_TYPE_FILE) { + type_str = type_str_file; + } + else if (type == MFU_TYPE_LINK) { + type_str = type_str_link; + } + + HDprintf("Type=%s File=%s\n", type_str, file); + } +} + +/* TODO: move this somewhere or modify existing print_file */ +/* print information about a file given the index and rank (used in print_files) */ +static size_t +print_file_text(mfu_flist flist, uint64_t idx, char *buffer, size_t bufsize) +{ + size_t numbytes = 0; + + /* store types as strings for print_file */ + char type_str_unknown[] = "UNK"; + char type_str_dir[] = "DIR"; + char type_str_file[] = "REG"; + char type_str_link[] = "LNK"; + + /* get filename */ + const char *file = mfu_flist_file_get_name(flist, idx); + + if (mfu_flist_have_detail(flist)) { + /* get mode */ + mode_t mode = (mode_t)mfu_flist_file_get_mode(flist, idx); + + uint64_t acc = mfu_flist_file_get_atime(flist, idx); + uint64_t mod = mfu_flist_file_get_mtime(flist, idx); + uint64_t cre = mfu_flist_file_get_ctime(flist, idx); + uint64_t size = mfu_flist_file_get_size(flist, idx); + const char *username = mfu_flist_file_get_username(flist, idx); + const char *groupname = mfu_flist_file_get_groupname(flist, idx); + + char access_s[30]; + char modify_s[30]; + char create_s[30]; + time_t access_t = (time_t)acc; + time_t modify_t = (time_t)mod; + time_t create_t = (time_t)cre; + size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t)); + size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t)); + size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t)); + if (access_rc == 0 || modify_rc == 0 || create_rc == 0) { + /* error */ + access_s[0] = '\0'; + modify_s[0] = '\0'; + create_s[0] = '\0'; + } + + char mode_format[11]; + mfu_format_mode(mode, mode_format); + + double size_tmp; + const char *size_units; + mfu_format_bytes(size, &size_tmp, &size_units); + + numbytes = (size_t)snHDprintf(buffer, bufsize, "%s %s %s %7.3f %3s %s %s\n", mode_format, username, + groupname, size_tmp, size_units, modify_s, file); + } + else { + /* get type */ + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + char * type_str = type_str_unknown; + if (type == MFU_TYPE_DIR) { + type_str = type_str_dir; + } + else if (type == MFU_TYPE_FILE) { + type_str = type_str_file; + } + else if (type == MFU_TYPE_LINK) { + type_str = type_str_link; + } + + numbytes = (size_t)snHDprintf(buffer, bufsize, "Type=%s File=%s\n", type_str, file); + } + + return numbytes; +} +#endif + +static size_t +get_local_bufsize(uint64_t *bufsize) +{ + size_t total = 0; + if (buft_count > 0) { + buf_t *lastbuf = buf_cache[buft_count - 1]; + size_t remaining = lastbuf->count; + total = (lastbuf->bufsize * buft_count) - remaining; + *bufsize = (uint64_t)(lastbuf->bufsize); + } + return total; +} + +static void +dh5tool_flist_write_text(const char *name, mfu_flist bflist) +{ + /* convert handle to flist_t */ + flist_t *flist = (flist_t *)bflist; + + /* get our rank and size of the communicator */ + int rank, ranks; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &ranks); + + /* start timer */ + double start_write = MPI_Wtime(); + + /* total list items */ + uint64_t all_count = mfu_flist_global_size(flist); + + /* report the filename we're writing to */ + if (mfu_rank == 0) { + MFU_LOG(MFU_LOG_INFO, "Writing to output file: %s", name); + } + + uint64_t idx = 0; + char * ptr = NULL; + + /* if we block things up into 128MB chunks, how many iterations + * to write everything? */ + // uint64_t maxwrite = 128 * 1024 * 1024; + uint64_t maxwrite = 0; + size_t local_total = get_local_bufsize(&maxwrite); + uint64_t iters = 0; + if (local_total > 0) + iters = (uint64_t)local_total / maxwrite; + + if (iters * maxwrite < (uint64_t)local_total) { + iters++; + } + + /* get max iterations across all procs */ + uint64_t all_iters; + MPI_Allreduce(&iters, &all_iters, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); + + /* use mpi io hints to stripe across OSTs */ + MPI_Info info; + MPI_Info_create(&info); + + /* change number of ranks to string to pass to MPI_Info */ + char str_buf[12]; + HDprintf(str_buf, "%d", ranks); + + /* no. of I/O devices for lustre striping is number of ranks */ + MPI_Info_set(info, "striping_factor", str_buf); + + /* open file */ + MPI_Status status; + MPI_File fh; + const char *datarep = "native"; + int amode = MPI_MODE_WRONLY | MPI_MODE_CREATE; + + int mpirc = MPI_File_open(MPI_COMM_WORLD, (const char *)name, amode, info, &fh); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to open file for writing: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* truncate file to 0 bytes */ + mpirc = MPI_File_set_size(fh, 0); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to truncate file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* set file view to be sequence of datatypes past header */ + mpirc = MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, datarep, MPI_INFO_NULL); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to set view on file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* compute byte offset to write our element */ + uint64_t offset = 0; + uint64_t bytes = (uint64_t)local_total; + MPI_Exscan(&bytes, &offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); + MPI_Offset write_offset = (MPI_Offset)offset; + + uint64_t written = 0; + while (all_iters > 0) { + /* compute number of bytes left to write */ + uint64_t remaining = (uint64_t)local_total - written; + + /* maybe Incr pointer to our next buffer */ + if (remaining == 0) { + idx++; + if (buf_cache[idx]->buf == NULL) { + } + } + + /* compute count we'll write in this iteration */ + int write_count = (int)maxwrite; + if (remaining < maxwrite) { + write_count = (int)remaining; + } + /* Get the buffer to output to the selected file */ + ptr = buf_cache[idx]->buf; + + /* collective write of file data */ + mpirc = MPI_File_write_at_all(fh, write_offset, ptr, write_count, MPI_BYTE, &status); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to write to file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* update our offset into the file */ + write_offset += (MPI_Offset)write_count; + + /* update number of bytes written so far */ + written += (uint64_t)write_count; + + /* update pointer into our buffer */ + ptr += write_count; + + /* decrement our collective write loop counter */ + all_iters--; + } + + /* free buffer */ + // mfu_free(&buf); + + /* close file */ + mpirc = MPI_File_close(&fh); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to close file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* free mpi info */ + MPI_Info_free(&info); + + /* end timer */ + double end_write = MPI_Wtime(); + + /* report write count, time, and rate */ + if (mfu_rank == 0) { + double secs = end_write - start_write; + double rate = 0.0; + if (secs > 0.0) { + rate = ((double)all_count) / secs; + } + MFU_LOG(MFU_LOG_INFO, "Wrote %lu files in %.3lf seconds (%.3lf files/sec)", all_count, secs, rate); + } + + return; +} + +static void +filter_hdf_files(mfu_flist *pflist, char *regex_exp, int exclude, int name) +{ + mfu_flist flist = *pflist; + mfu_flist eligible = mfu_flist_subset(flist); + uint64_t idx = 0; + uint64_t files = mfu_flist_size(flist); + while (idx < files) { + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK || type == MFU_TYPE_UNKNOWN) { + const char *file = mfu_flist_file_get_name(flist, idx); + int accessible = H5Fis_accessible(file, H5P_DEFAULT); + if (accessible) + mfu_flist_file_copy(flist, idx, eligible); + } + idx++; + } + + mfu_flist_summarize(eligible); + + /* assume we'll use the full list */ + // mfu_flist srclist = flist; + mfu_flist srclist = eligible; + + /* filter the list if needed */ + mfu_flist filtered_flist = MFU_FLIST_NULL; + if (regex_exp != NULL) { + /* filter the list based on regex */ + filtered_flist = mfu_flist_filter_regex(eligible, regex_exp, exclude, name); + + /* update our source list to use the filtered list instead of the original */ + srclist = filtered_flist; + } + + mfu_flist_free(&flist); + *pflist = srclist; + return; +} + +static int +fill_file_list(mfu_flist new_flist, const char *config_filename, int myrank, int size) +{ + int index = 0; + char linebuf[PATH_MAX] = { + '\0', + }; + FILE *config = HDfopen(config_filename, "r"); + if (config == NULL) + return -1; + while (HDfgets(linebuf, sizeof(linebuf), config) != NULL) { + struct stat statbuf; + char * eol = HDstrchr(linebuf, '\n'); + if (eol) + *eol = '\0'; + if (HDstat(linebuf, &statbuf) == 0) { + if (myrank == (index % size)) { + mfu_flist_insert_stat((flist_t *)new_flist, linebuf, O_RDONLY, &statbuf); + } + index++; + } + linebuf[0] = 0; + } + HDfclose(config); + return index; +} + +static int +count_dirpaths(int argc, int startcnt, const char *argv[], int **index_out) +{ + int k; + int path_cnt = 0; + int idx_count = (argc - startcnt); + int * index = NULL; + struct stat pathcheck; + + if (idx_count > 0) { + index = (int *)malloc((size_t)(argc - startcnt) * sizeof(int)); + assert(index); + } + else + return 0; + + for (k = startcnt; k < argc; k++) { + char *slash = NULL; + int c = *argv[k]; + if ((c == '.') || (c == '/')) { + index[path_cnt++] = k; + } + else if ((c == '@')) { + const char *configFile = argv[k] + 1; + if (stat(configFile, &pathcheck) == 0) { + if (S_ISREG(pathcheck.st_mode)) { + config_index[use_config_file++] = k; + } + } + } + else if ((slash = strchr(argv[k], '/')) != NULL) { + if (stat(argv[k], &pathcheck) == 0) { + if (S_ISDIR(pathcheck.st_mode)) + index[path_cnt++] = k; + } + } + } + if ((path_cnt == 0) && (index != NULL)) { + free(index); + return 0; + } + *index_out = index; + return path_cnt; +} + +static char ** +copy_args(int argc, const char *argv[], int *mfu_argc, int *copy_len) +{ + int i, bytes_copied = 0; + int check_mfu_args = 1; + char **argv_copy = (char **)MFU_MALLOC((size_t)(argc + 2) * sizeof(char **)); + assert(argv_copy); + assert(mfu_argc); + assert(copy_len); + save_command(argv[0]); + + for (i = 0; i < argc; i++) { + argv_copy[i] = HDstrdup(argv[i]); + bytes_copied += (int)(strlen(argv[i]) + 1); + argv_copy[i] = HDstrdup(argv[i]); + if (check_mfu_args && (HDstrncmp(argv[i], "-T", 2) == 0)) { + check_mfu_args = 0; + *mfu_argc = i + 1; + } + } + argv_copy[i] = 0; + *copy_len = bytes_copied; + return argv_copy; +} + +typedef struct hash_entry { + int hash; + char * name; + struct hash_entry *next; /* table Collision */ + int nextCount; +} hash_entry_t; + +#ifndef NAME_ENTRIES +#define NAME_ENTRIES 4096 +#endif + +static hash_entry_t filename_cache[NAME_ENTRIES]; + +static int +get_copy_count(char *fname, char *appname) +{ + int filehash = 0, apphash = 0; + size_t k, applen = strlen(appname); + size_t filelen = strlen(fname); + int hash_index; + + for (k = 0; k < filelen; k++) { + filehash += fname[k]; + } + for (k = 0; k < applen; k++) { + apphash += appname[k]; + } + hash_index = filehash % NAME_ENTRIES; + if (filename_cache[hash_index].name == NULL) { + filename_cache[hash_index].hash = apphash; + filename_cache[hash_index].name = HDstrdup(fname); + filename_cache[hash_index].next = NULL; + filename_cache[hash_index].nextCount = 1; + return 0; + } + else if ((apphash == filename_cache[hash_index].hash) && + (strcmp(filename_cache[hash_index].name, fname) == 0)) { + int retval = filename_cache[hash_index].nextCount++; + return retval; + } + else { /* Collision */ + hash_entry_t *nextEntry = &filename_cache[hash_index]; + hash_entry_t *lastEntry = nextEntry; + while (nextEntry) { + if ((apphash == nextEntry->hash) && (strcmp(nextEntry->name, fname) == 0)) { + /* Match (increment nextCount and return) */ + int retval = nextEntry->nextCount++; + return retval; + } + else { + /* No Match (continue search) */ + lastEntry = nextEntry; + nextEntry = lastEntry->next; + } + } + nextEntry = (hash_entry_t *)malloc(sizeof(hash_entry_t)); + if (nextEntry) { + lastEntry->next = nextEntry; + nextEntry->name = HDstrdup(fname); + nextEntry->hash = apphash; + nextEntry->next = NULL; + nextEntry->nextCount = 1; + } + } + return 0; +} + +static void +run_command(int argc __attribute__((unused)), char **argv, char *cmdline, const char *fname) +{ + char filepath[1024]; + char *toolname = argv[0]; + char *buf = NULL; + int use_stdout = 0; + +#ifdef H5_HAVE_WINDOWS + HDprintf("ERROR: %s %s: Unable to support fork/exec on WINDOWS\n", PROGRAMNAME, __func__); + h5dwalk_exit(EXIT_FAILURE); +#else + + /* create a copy of the 1st file passed to the application */ + HDstrcpy(filepath, fname); + + if (log_output_in_single_file || use_stdout) { + pid_t pid; + int pipefd[2]; + buf_t * thisbuft = NULL; + buf_t **bufs = buf_cache; + + if (bufs == NULL) { + bufs = (buf_t **)MFU_CALLOC(buft_max, sizeof(buf_t *)); + assert((bufs != NULL)); + buf_cache = bufs; +#ifdef VERBOSE + if (buft_count == 0) { + HDprintf("[%d] Initial buf_cache allocation: buft_count=%d\n", sg_mpi_rank, buft_count); + } +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert((thisbuft != NULL)); + } + else { + thisbuft = bufs[buft_count - 1]; + assert((thisbuft != NULL)); + /* Check for remaining space in the current buffer */ + /* If none, then create a new buffer */ + if (thisbuft->count == 0) { + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + } + } + if ((thisbuft->buf == NULL)) { + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + assert((thisbuft->buf != NULL)); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->count = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + } + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + pid = fork(); + if (pid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + if (pid == 0) { + close(pipefd[0]); + dup2(pipefd[1], fileno(stdout)); + dup2(pipefd[1], fileno(stderr)); + execvp(argv[0], argv); + } + else { + int w_status; + size_t nbytes; + size_t read_bytes = 0; + uint64_t remaining, offset; + close(pipefd[1]); + buf = thisbuft->buf; + remaining = thisbuft->count; + offset = thisbuft->chars; + nbytes = strlen(cmdline); + /* Record the command line for the log! */ + if (nbytes < remaining) { + HDstrcpy(&buf[offset], cmdline); + thisbuft->chars += nbytes; + thisbuft->count -= nbytes; + remaining -= nbytes; + } + else { /* We're running out of space in the current buffer */ + char *nextpart; + strncpy(&buf[offset], cmdline, remaining); + nextpart = &cmdline[remaining + 1]; + thisbuft->count = 0; + thisbuft->chars += remaining; + + /* Create a new read buffer */ +#ifdef VERBOSE + HDprintf("[%d] Allocate-1 a new read buffer:: buft_count=%d\n", sg_mpi_rank, buft_count); +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert(thisbuft != NULL); + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + /* Copy the remaining cmdline text into the new buffer */ + HDstrcpy(buf, nextpart); + /* And update our buffer info */ + // thisbuft->chars = strlen(nextpart) +1; + thisbuft->chars = strlen(nextpart); + thisbuft->count = BUFT_SIZE - thisbuft->chars; + } + offset = thisbuft->chars; + + do { + waitpid(pid, &w_status, WNOHANG); + if ((nbytes = (size_t)read(pipefd[0], &buf[offset], remaining)) > 0) { + offset += nbytes; + read_bytes += nbytes; + remaining -= nbytes; + if (remaining == 0) { + /* Update the current buffer prior to allocating the new one */ + thisbuft->count = 0; + thisbuft->chars += read_bytes; +#ifdef VERBOSE + HDprintf("[%d] Allocate-2 a new read buffer:: buft_count=%d\n", sg_mpi_rank, + buft_count); +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert(thisbuft != NULL); + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + thisbuft->chars = BUFT_SIZE; + offset = 0; + remaining = BUFT_SIZE; + } + } + } while (!WIFEXITED(w_status)); + close(pipefd[0]); + wait(NULL); + + thisbuft->count = remaining; + thisbuft->chars = thisbuft->bufsize - remaining; + } + } + else if (log_stdout_in_file) { + int log_instance = -1; + pid_t pid; + size_t log_len; + char logpath[2048]; + char logErrors[2048]; + char current_dir[2048]; + char * logbase = HDstrdup(basename(filepath)); + char * thisapp = HDstrdup(basename(toolname)); + + if (processing_inputfile == 0) + log_instance = get_copy_count(logbase, thisapp); + + if (txtlog == NULL) { + if ((log_instance > 0) || processing_inputfile) { + if (processing_inputfile) + log_instance = current_input_index; + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log_%d", + HDgetcwd(current_dir, sizeof(current_dir)), logbase, thisapp, log_instance); + } + else { + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log", + HDgetcwd(current_dir, sizeof(current_dir)), logbase, thisapp); + } + } + else { + log_len = strlen(txtlog); + if ((log_instance > 0) || processing_inputfile) { + if (processing_inputfile) + log_instance = current_input_index; + if (txtlog[log_len - 1] == '/') + HDsnprintf(logpath, sizeof(logpath), "%s%s_%s.log_%d", txtlog, logbase, thisapp, + log_instance); + else + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log_%d", txtlog, logbase, thisapp, + log_instance); + } + else { + if (txtlog[log_len - 1] == '/') + HDsnprintf(logpath, sizeof(logpath), "%s%s_%s.log", txtlog, logbase, thisapp); + else + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log", txtlog, logbase, thisapp); + } + } + + if (log_errors_in_file) { + /* We co-locate the error logs in the same directories as the regular log files. + * The easiest way to do this is to simply replace the .log with .err in a + * copy of the logpath variable. + */ + log_len = strlen(logpath); + HDstrcpy(logErrors, logpath); + HDstrcpy(&logErrors[log_len - 3], "err"); + } + if (mfu_debug_level == MFU_LOG_VERBOSE) { + HDprintf("\tCreating logfile: %s\n", logpath); + fflush(stdout); + } + pid = fork(); + if (pid == 0) { + int efd; + int fd = open(logpath, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + dup2(fd, fileno(stdout)); + if (log_errors_in_file) { + efd = open(logErrors, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + dup2(efd, fileno(stderr)); + close(efd); + } + else + dup2(fd, fileno(stderr)); + close(fd); + execvp(argv[0], argv); + } + int status; + pid = wait(&status); + if (logbase) + free(logbase); + if (thisapp) + free(thisapp); + } /* else if(log_stdout_in_file) */ +#endif /* #ifdef H5_HAVE_WINDOWS */ +} + +int MFU_PRED_EXEC(mfu_flist flist, uint64_t idx, void *arg); +int MFU_PRED_PRINT(mfu_flist flist, uint64_t idx, void *arg); + +int +MFU_PRED_EXEC(mfu_flist flist, uint64_t idx, void *arg) +{ + /* get file name for this item */ + int file_substituted = 0; + const char *fname = mfu_flist_file_get_name(flist, idx); + + char *toolname = NULL; + char filepath[1024]; + + size_t b_offset; + + /* get pointer to encoded argc count and argv array */ + int * count_ptr = arg; + char *buf = (char *)arg + sizeof(int); + + /* get number of argv parameters */ + int k = 0, count = *count_ptr; + toolname = buf; + + /* Get a copy of fname */ + HDstrcpy(filepath, fname); + + /* allocate a char* for each item in the argv array, + * plus one more for a trailing NULL + * 'count' in this case is the number of args, so + * so we add (+1) for the toolname and another (+1) + * for the trailing NULL to terminate the list + */ + + char cmdline[2048]; + char **argv = (char **)MFU_CALLOC((size_t)(count + 2), sizeof(char *)); + + argv[k++] = HDstrdup(toolname); + + HDmemset(cmdline, 0, sizeof(cmdline)); + buf += HDstrlen(toolname) + 1; + /* Reconstruct the command line that the user provided for the h5tool */ + for (k = 1; k < count; k++) { + if (buf[0] == '&') { + const char *fname_arg = NULL; + mfu_flist flist_arg; + void * check_ptr[2] = {NULL, NULL}; + + HDmemcpy(check_ptr, &buf[1], sizeof(void *)); + flist_arg = (mfu_flist)check_ptr[0]; + + /* +2 (see below) accounts for the '&' and the trailing zero pad */ + buf += sizeof(mfu_flist *) + 2; + fname_arg = mfu_flist_file_get_name(flist_arg, idx); + if (fname_arg == NULL) { + HDprintf("[%d] Warning: Unable to resolve file_substitution %d (idx=%ld)\n", sg_mpi_rank, + file_substituted, idx); + argv[k] = HDstrdup(fname); + } + else { + argv[k] = HDstrdup(fname_arg); + file_substituted++; + } + } + else { + argv[k] = HDstrdup(buf); + buf += HDstrlen(argv[k]) + 1; + } + } + + HDsnprintf(cmdline, sizeof(cmdline), "\n---------\nCommand:"); + b_offset = strlen(cmdline); + for (k = 0; k < count; k++) { + HDsprintf(&cmdline[b_offset], " %s", argv[k]); + b_offset = strlen(cmdline); + } + HDsprintf(&cmdline[b_offset], "\n"); + run_command(count, argv, cmdline, fname); + + mfu_free(argv); + + return 0; +} + +int +MFU_PRED_PRINT(mfu_flist flist, uint64_t idx, void *arg __attribute__((unused))) +{ + const char *name = mfu_flist_file_get_name(flist, idx); + HDprintf("%s\n", name); + return 1; +} + +static void +pred_commit(mfu_pred *p) +{ + mfu_pred *cur = p; + while (cur) { + if (cur->f == MFU_PRED_PRINT || cur->f == MFU_PRED_EXEC) { + break; + } + cur = cur->next; + } +} + +static void +add_executable(int argc, char **argv, char *cmdstring, int *f_index, int f_count __attribute__((unused))) +{ + char cmdline[2048]; + HDsnprintf(cmdline, sizeof(cmdline), "\n---------\nCommand: %s\n", cmdstring); + argv[argc] = NULL; + run_command(argc, argv, cmdline, argv[f_index[0]]); + return; +} + +static int +process_input_file(char *inputname, int myrank, int size) +{ + int index = 0; + char linebuf[PATH_MAX] = { + '\0', + }; + FILE * config = HDfopen(inputname, "r"); + mfu_flist flist1 = NULL; + + if (config == NULL) + return -1; + + flist1 = mfu_flist_new(); + + /* Flag the fact that we're processing an inputfile (script) + * so that we can generate a meaningful logfile name... + */ + processing_inputfile = 1; + + while (HDfgets(linebuf, sizeof(linebuf), config) != NULL) { + const char *delim = " \n"; + char * cmdline = NULL; + char * cmd = NULL; + char * arg = NULL; + char * argv[256]; + int fileindex[256]; + int filecount = 0; + int token = 0; + struct stat statbuf; + + char *eol = strchr(linebuf, '\n'); + if (eol) { + *eol = '\0'; + } + cmdline = HDstrdup(linebuf); + cmd = HDstrtok(linebuf, delim); + if (cmd) { + arg = cmd; + while (arg != NULL) { + char c = arg[0]; + if (token > 0) { + if ((c == '.') || (c == '/')) { + /* 'arg' looks to be a filepath */ + if (stat(arg, &statbuf) == 0) { + mfu_flist_insert_stat(flist1, arg, O_RDONLY, &statbuf); + } + fileindex[filecount++] = token; + } + } + argv[token++] = arg; + arg = strtok(NULL, delim); + } + + if (myrank == (index % size)) { + current_input_index = index; + add_executable(token, argv, cmdline, fileindex, filecount); + } + index++; + } + linebuf[0] = 0; + HDfree(cmdline); + } + + if (output_log_file) { + dh5tool_flist_write_text(output_log_file, flist1); + } + HDfclose(config); + + mfu_flist_free(&flist1); + return 0; +} + +int +main(int argc, char *argv[]) +{ + int i; + int rc = 0; + + char *env_var = NULL; + + /* initialize MPI */ + MPI_Init(&argc, (char ***)&argv); + mfu_init(); + + /* Initialize h5tools lib */ + h5tools_init(); + + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + + /* get our rank and the size of comm_world */ + int rank, ranks; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &ranks); + + /* Assign the static global mpi_rank (for debugging) */ + sg_mpi_rank = rank; + +#if 0 + env_var = HDgetenv("HDF5_H5DWALK_PRINT_CMDLINE"); + if (env_var) { + int enable = HDatoi(env_var); + if (enable) { + + } + } +#endif + /* pointer to mfu_walk_opts */ + mfu_walk_opts_t *walk_opts = mfu_walk_opts_new(); + +#ifdef DAOS_SUPPORT + /* DAOS vars */ + daos_args_t *daos_args = daos_args_new(); +#endif + + int args_byte_length = -1; + int mfu_argc = argc; + char * args_buf = NULL; + char **h5tool_argv = copy_args(argc, argv, &mfu_argc, &args_byte_length); + + char *inputname = NULL; + char *outputname = NULL; + char *sortfields = NULL; + char *distribution = NULL; + + int text = 0; + int h5tool_argc = 0; + + mfu_debug_level = MFU_LOG_WARN; + h5tool_argv[argc] = 0; + + /* The struct option declaration can found in bits/getopt_ext.h + * I've reproduced it here: + * struct option { char * name; int has_arg; int *flag; int val}; + */ + int opt; + int tool_selected = 0; + int tool_args_start = -1; + int last_mfu_arg = 0; + + mfu_pred *pred_head = NULL; + + while (!tool_selected) { + opt = H5_get_option(argc, (const char *const *)argv, s_opts, l_opts); + switch ((char)opt) { + default: + usage(); + h5dwalk_exit(EXIT_FAILURE); + break; + case 'i': + inputname = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + if (inputname) + tool_selected = 1; + break; + case 'o': + outputname = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + if (outputname) { + log_output_in_single_file = 1; + output_log_file = HDstrdup(H5_optarg); + text = 1; /* Format TXT, not HDF5 */ + } + break; + case 'E': + log_errors_in_file = 1; + errlog = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + break; + case 'l': + log_stdout_in_file = 1; + if (H5_optarg) + txtlog = HDstrdup(H5_optarg); + break; + case 'T': + /* We need to stop parsing user options at this point. + * all remaining arguments should be utilized as the + * arguments to the selected HDF5 tools. + * We also want to avoid any misinterpretations if + * HDF5 tool options conflict with the MFU options. + */ + tool_selected = 1; + tool_args_start = H5_optind; + h5tool_argc = argc - mfu_argc; + last_mfu_arg = H5_optind; + /* Don't allow any further parsing of arguments */ + break; + case 'h': + usage(); + h5dwalk_exit(EXIT_SUCCESS); + break; + case '?': + usage(); + h5dwalk_exit(EXIT_SUCCESS); + break; + } + } + + if (inputname != NULL) { + if (tool_selected && (rank == 0)) { + if ((log_output_in_single_file == 0) && (log_stdout_in_file == 0)) + puts("WARNING: When utilizing --input, the only other supported " + "runtime argument is --output or -l"); + } + rc = process_input_file(inputname, rank, ranks); + mfu_finalize(); + h5dwalk_exit(rc); + } + + /**************************************************************/ + /* We might consider doing a tool specific argument checking */ + /* to prevent runtime errors. We would also like to allow */ + /* the same command line interface for parallel invocations */ + /* so that users don't get confused. Effectively, we should */ + /* strip out all MFU related arguments and retain copies of */ + /* everything else to pass into a serial instance of the tool */ + /* */ + /* As we move forward, we might allow the HDF5 tool to be */ + /* queried for an acceptable set set of runtime arguments. */ + /* This could be just a simple string to allow getopt_long */ + /* to be invoked on the remaining command line arguments. */ + /**************************************************************/ + + int *path_indices = NULL; + int numpaths = count_dirpaths(argc, tool_args_start, argv, &path_indices); + + const char **argpaths = NULL; + + /* store src and dest path strings */ + const char *path1 = NULL; + const char *path2 = NULL; + size_t pathlen_total = 0; + + if (numpaths && path_indices) { + argpaths = &argv[path_indices[0]]; + } + /* pointer to mfu_file src and dest objects */ + /* The dst object will only be used for tools which + * accept 2 (or more?) file arguments */ + mfu_file_t *mfu_src_file = NULL; + mfu_file_t *mfu_dst_file = NULL; + + /* first item is source and second is dest */ + mfu_param_path *srcpath = NULL; + mfu_param_path *destpath = NULL; + mfu_param_path *paths = NULL; + + mfu_flist flist1 = NULL; + mfu_flist flist2 = NULL; + + /* allocate structure to define walk options */ + if (use_config_file > 0) { + int count1 = 0, count2 = 0; + for (i = 0; i < use_config_file; i++) { + int index = config_index[i]; + const char *config_file = argv[index]; + if (i == 0) { + flist1 = mfu_flist_new(); + count1 = fill_file_list(flist1, config_file + 1, rank, ranks); + } + else if (i == 1) { + flist2 = mfu_flist_new(); + count2 = fill_file_list(flist2, config_file + 1, rank, ranks); + } + } + if (count1 != count2) { + HDprintf("config files have different file counts: (1) %d and (2) %d\n", count1, count2); + } + } + else if (numpaths > 0) { + + /* allocate space for each path */ + paths = (mfu_param_path *)MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path)); + mfu_src_file = mfu_file_new(); + + /* process each path */ + mfu_param_path_set_all((uint64_t)numpaths, (const char **)argpaths, paths, mfu_src_file, true); + + /* don't allow user to specify input file with walk */ + if (inputname != NULL) { + if (paths) { + mfu_free(&paths); + } + usage(); + h5dwalk_exit(EXIT_FAILURE); + } + } + else { + /* if we're not walking, we must be reading, + * and for that we need a file */ + if (inputname == NULL) { + if (rank == 0) { + MFU_LOG(MFU_LOG_ERR, "Either a <path> or --input is required."); + } + usage(); + h5dwalk_exit(EXIT_FAILURE); + } + } + + if (numpaths > 0) { + flist1 = mfu_flist_new(); + srcpath = &paths[0]; + path1 = srcpath->path; + pathlen_total += strlen(path1); + mfu_flist_walk_param_paths(1, srcpath, walk_opts, flist1, mfu_src_file); + } + if (numpaths > 1) { + flist2 = mfu_flist_new(); + mfu_dst_file = mfu_file_new(); + destpath = &paths[1]; + path2 = destpath->path; + pathlen_total += HDstrlen(path2); + mfu_flist_walk_param_paths(1, destpath, walk_opts, flist2, mfu_dst_file); + } + + if (tool_selected && (args_byte_length > 0)) { + pred_head = mfu_pred_new(); + args_buf = (char *)HDmalloc((size_t)(args_byte_length + pathlen_total)); + } + + /* filter files to only include hdf5 files */ + if (flist1) { + filter_hdf_files(&flist1, NULL, 0, 0); + } + if (flist2) { + filter_hdf_files(&flist2, NULL, 0, 0); + } + + /* if (numpaths > 1) + * In a case where we requeire the list indices of files from multiple + * directories to match, we must utilize a mapping function. + * The question to answer is how does the mapping function work? + * The most probable is a sort function, e.g. + * 1) an alphabet sort? + * 2) sort by file size? + * 3) something else? + */ + if (args_buf != NULL) { + int k = 0; + char *ptr = args_buf + sizeof(int); + *(int *)args_buf = h5tool_argc; + for (i = tool_args_start - 1; i < argc; i++) { + int copy_flist = -1; + if (i == config_index[k]) { + copy_flist = k; + } + else if (path_indices && (i == path_indices[k])) { + copy_flist = k; + } + + /* Maybe copy one of the flist pointers */ + if (copy_flist >= 0) { + /* The '&' indicates that what follows is a pointer */ + *ptr++ = '&'; + /* Select which argument list should be used */ + if (k == 0) { + HDmemcpy(ptr, &flist1, sizeof(void *)); + } + if (k == 1) { + HDmemcpy(ptr, &flist2, sizeof(void *)); + } + ptr += sizeof(mfu_flist *); + k++; + } + else { + HDstrcpy(ptr, argv[i]); + ptr += HDstrlen(argv[i]); + } + *ptr++ = 0; + } + *ptr++ = 0; + + mfu_pred_add(pred_head, MFU_PRED_EXEC, (void *)args_buf); + pred_commit(pred_head); + } + + /* apply predicates to each item in list */ + mfu_flist flist3 = mfu_flist_filter_pred(flist1, pred_head); + + /* print summary statistics of flist */ + mfu_flist_print_summary(flist1); + + /* write data to cache file */ + if (outputname != NULL) { + if (!text) { + if (rank == 0) { + puts("output capture needs to be a text formatted file"); + } + } + else { + dh5tool_flist_write_text(outputname, flist1); + } + } + +#ifdef DAOS_SUPPORT + daos_cleanup(daos_args, mfu_file, NULL); +#endif + + /* free users, groups, and files objects */ + mfu_flist_free(&flist1); + if (flist2) + mfu_flist_free(&flist2); + if (flist3) + mfu_flist_free(&flist3); + + /* free memory allocated for options */ + mfu_free(&distribution); + mfu_free(&sortfields); + mfu_free(&outputname); + mfu_free(&inputname); + + /* free the path parameters */ + mfu_param_path_free_all((uint64_t)numpaths, paths); + + /* free memory allocated to hold params */ + mfu_free(&paths); + + /* free the walk options */ + mfu_walk_opts_delete(&walk_opts); + + /* delete file object */ + mfu_file_delete(&mfu_src_file); + + h5tools_close(); + /* shut down MPI */ + mfu_finalize(); + MPI_Finalize(); + + return rc; +} + +/*------------------------------------------------------------------------- + * Function: h5dwalk_exit + * + * Purpose: close the tools library and exit + * + * Return: none + * + * Programmer: Albert Cheng + * Date: Feb 6, 2005 + * + * Comments: + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +H5_ATTR_NORETURN void +h5dwalk_exit(int status) +{ + int require_finalize = 0; + h5tools_close(); + mfu_finalize(); + + /* Check to see whether we need to call MPI_Finalize */ + MPI_Initialized(&require_finalize); + if (require_finalize) + MPI_Finalize(); + + HDexit(status); +} diff --git a/utils/tools/test/CMakeLists.txt b/utils/tools/test/CMakeLists.txt new file mode 100644 index 0000000..0f5335d --- /dev/null +++ b/utils/tools/test/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_TEST C) + +#-- Add the h5diff tests +if (HDF5_BUILD_PARALLEL_TOOLS) + add_subdirectory (h5dwalk) +endif() + diff --git a/utils/tools/test/Makefile.am b/utils/tools/test/Makefile.am new file mode 100644 index 0000000..88104f2 --- /dev/null +++ b/utils/tools/test/Makefile.am @@ -0,0 +1,32 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Tools HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +if PARALLEL_TOOLS_CONDITIONAL + H5DWALK=h5dwalk +else + H5DWALK= +endif + +CONFIG=ordered + +# All subdirectories +SUBDIRS=$(H5DWALK) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/test/h5dwalk/CMakeLists.txt b/utils/tools/test/h5dwalk/CMakeLists.txt new file mode 100644 index 0000000..5f6c992 --- /dev/null +++ b/utils/tools/test/h5dwalk/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_TEST_H5DWALK) + +if (HDF5_BUILD_PARALLEL_TOOLS) + add_custom_command( + OUTPUT ${HDF5_TOOLS_DIR}/test/demo_destfiles.test + COMMAND bash -c ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh + ARGS ${HDF5_TOOLS_DIR}/test ${CMAKE_BINARY_DIR}/bin + DEPENDS ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh + ) +endif () + +if (HDF5_TEST_TOOLS AND HDF5_TEST_SERIAL) + include (CMakeTests.cmake) +endif () diff --git a/utils/tools/test/h5dwalk/CMakeTests.cmake b/utils/tools/test/h5dwalk/CMakeTests.cmake new file mode 100644 index 0000000..b9e52c5 --- /dev/null +++ b/utils/tools/test/h5dwalk/CMakeTests.cmake @@ -0,0 +1,56 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_TEST resultfile resultcode) + # If using memchecker add tests without using scripts + if (HDF5_ENABLE_USING_MEMCHECKER) + message("Entered ADD_H5_TEST - 0") + add_test (NAME H5DWALK-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5dwalk${tgt_file_ext}> ${ARGN}) + set_tests_properties (H5DWALK-${resultfile} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if ("${resultcode}" STREQUAL "1") + set_tests_properties (H5DWALK-${resultfile} PROPERTIES WILL_FAIL "true") + endif () + else () + # Remove any output file left over from previous test run + add_test ( + NAME H5DWALK-${resultfile} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5dwalk${tgt_file_ext}>" + -D "TEST_ARGS=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.h5dwalk" + -D "TEST_LIBRARY_DIRECTORY=${LL_PATH}" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + endif () + endmacro () + + ADD_H5_TEST(help-1 0 -h) diff --git a/utils/tools/test/h5dwalk/Makefile.am b/utils/tools/test/h5dwalk/Makefile.am new file mode 100644 index 0000000..c32dd0f --- /dev/null +++ b/utils/tools/test/h5dwalk/Makefile.am @@ -0,0 +1,43 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src directory +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib + +install-examples: + @echo "Creating demo files" && \ + . copy_demo_files.sh $(top_srcdir)/src + +bin_SCRIPTS:install-examples + +#test script and program +TEST_SCRIPT=testh5dwalk.sh copy_demo_files.sh +check_SCRIPTS=$(TEST_SCRIPT) +SCRIPT_DEPEND=../../h5dwalk/h5dwalk$(EXEEXT) + +# Tell automake to clean h5redeploy script +CLEANFILES= + +# These were generated by configure. Remove them only when distclean. +DISTCLEANFILES=testh5dwalk.sh copy_demo_files.sh + +# All programs rely on hdf5 library and h5tools library +LDADD=$(LIBH5TOOLS) $(LIBHDF5) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/test/h5dwalk/copy_demo_files.sh.in b/utils/tools/test/h5dwalk/copy_demo_files.sh.in new file mode 100644 index 0000000..f20bf43 --- /dev/null +++ b/utils/tools/test/h5dwalk/copy_demo_files.sh.in @@ -0,0 +1,86 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +srcdir=@srcdir@ +TOP_BUILDDIR=.. + +# Determine if backward compatibility options enabled +DEPRECATED_SYMBOLS="yes" + +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +CP='cp' + +THIS_DIR=`pwd` +SRC_TOOLS_DIR=$srcdir/../../../../tools + +nerrors=0 +verbose=yes +exit_code=$EXIT_SUCCESS + + +# Add Testing files into the local testfiles directory:: +TESTDIR=./testfiles +test -d $TESTDIR || mkdir $TESTDIR + +echo "HDF5 \"$THIS_DIR/testfiles/h5diff_basic1.h5\" {" > "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt +echo "FILE_CONTENTS { + group / + group /g1 + dataset /g1/d1 + dataset /g1/d2 + dataset /g1/dset1 + dataset /g1/dset10 + dataset /g1/dset11 + dataset /g1/dset12 + dataset /g1/dset3 + dataset /g1/dset5 + dataset /g1/dset6 + dataset /g1/dset7 + dataset /g1/dset8 + dataset /g1/dset9 + dataset /g1/fp1 + dataset /g1/fp15 + dataset /g1/fp16 + dataset /g1/fp17 + dataset /g1/fp18 + dataset /g1/fp18_COPY + dataset /g1/fp19 + dataset /g1/fp19_COPY + dataset /g1/fp2 + dataset /g1/fp20 + dataset /g1/fp20_COPY + dataset /g1/ld + } +}" >> "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt + +# Create the help-1.txt output file for '-h' validation +echo " +Usage: h5dwalk [options] <path> ... + +Options: + -i, --input <file> - read list from file + -o, --output <file> - write output summary to the named file. + -E, --error <file> - write processed errors to file in text format + -l, --log_text <dir> - write individual tool outputs to a file. Logs can be written to an optional named directory. + -T, --tool <executable> - name of the HDF5 tool to invoke + -h, --help - print usage + +For more information see https://mpifileutils.readthedocs.io. +" > "$THIS_DIR"/testfiles/help-1.txt + +# Make a copy of the help-1.txt output file for --help validation +$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-1.txt +$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-2.txt +# Make a copy of a simple HDF5 datafile which will be used as input for h5dump -n (see the expected output above) +$CP "$SRC_TOOLS_DIR"/test/h5diff/testfiles/h5diff_basic1.h5 "$THIS_DIR"/testfiles diff --git a/utils/tools/test/h5dwalk/help.h5dwalk b/utils/tools/test/h5dwalk/help.h5dwalk new file mode 100644 index 0000000..986cbba --- /dev/null +++ b/utils/tools/test/h5dwalk/help.h5dwalk @@ -0,0 +1,13 @@ + +Usage: h5dwalk [options] <path> ... + +Options: + -i, --input <file> - read list from file + -o, --output <file> - write output summary to the named file. + -E, --error <file> - write processed errors to file in text format + -l, --log_text <dir> - write individual tool outputs to a file. Logs can be written to an optional named directory. + -T, --tool <executable> - name of the HDF5 tool to invoke + -h, --help - print usage + +For more information see https://mpifileutils.readthedocs.io. + diff --git a/utils/tools/test/h5dwalk/testh5dwalk.sh.in b/utils/tools/test/h5dwalk/testh5dwalk.sh.in new file mode 100644 index 0000000..4f6dbde --- /dev/null +++ b/utils/tools/test/h5dwalk/testh5dwalk.sh.in @@ -0,0 +1,249 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +# Tests for the h5dwalk tool + +# +# + +srcdir=@srcdir@ + +# Determine which filters are available +USE_FILTER_SZIP="@USE_FILTER_SZIP@" +USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" + + +TESTNAME=h5dwalk + +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +THIS_DIR="`pwd`" +ROOTDIR="`cd ../../../..; pwd`" +cd "$THIS_DIR" +TOP_DIR="$ROOTDIR" + + +H5DWALK=../../h5dwalk/h5dwalk +H5DWALK_BIN="$TOP_DIR/utils/tools/h5dwalk/h5dwalk" + + +H5DUMP="$TOP_DIR/src/h5dump/h5dump" +H5DUMP_BIN="$TOP_DIR/tools/src/h5dump/h5dump" + +RM='rm -rf' +CMP='cmp -s' +DIFF='diff -c' +CP='cp' +DIRNAME='dirname' +LS='ls' +AWK='awk' +WC='wc' + +nerrors=0 +verbose=yes + +export LD_LIBRARY_PATH=@LL_PATH@ + +# source dirs +SRC_TOOLS="$TOP_DIR/tools/test" +SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" + +# testfiles source dirs for tools +SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles" +SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles" +SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles" +SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles" +SRC_H5DWALK_TESTFILES="$SRC_TOOLS/h5dwalk/testfiles" +SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles" + +TESTDIR=./testfiles +test -d $TESTDIR || mkdir $TESTDIR + +echo "SRC_H5DIFF_TESTFILES = $SRC_H5DIFF_TESTFILES" +echo "Creating demo files" +. ./copy_demo_files.sh + + + +CLEAN_TESTFILES_AND_TESTDIR() +{ + echo "cleaning logfiles" + $RM $TESTDIR/*log* +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# +TOOLTEST() { + expect="$TESTDIR/$1" + expect_err="$TESTDIR/`basename $1`.err" + actual="$TESTDIR/`basename $1`.out" + actual_err="$TESTDIR/`basename $1`.out.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + shift + + # Run test. + + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + ) 1> $actual 2> $actual_err + + # save actual and actual_err in case they are needed later. + cp $actual $actual_sav + cp $actual_err $actual_err_sav + + if [ ! -f $expect ]; then + # Compare error files if the expect file doesn't exist. + if $CMP $expect_err $actual_err; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.err) differs from actual result (*.out.err)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect_err $actual_err |sed 's/^/ /' + fi + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav + fi +} + +TOOL_LOGTEST() { + expect="$TESTDIR/`basename $1`.txt" + expect_err="$TESTDIR/`basename $1`.err" + actual="$TESTDIR/`basename $1`.log" + actual_err="$TESTDIR/`basename $1`.out.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + shift + + echo "running logtest" + + # Run test. + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + + ) 1> $actual 2> $actual_err + expect_len="`wc -l < $expect`" + + if [ ! -f $actual ]; then + echo "*FAILED*" + echo " The expected .log file is missing" + echo " Perhaps the test failed to run?" + else + actual_len="`wc -l < $actual`" + if [ $actual_len -eq $expect_len ]; then + echo " PASSED" + else + echo "*FAILED*" + echo " The generated .log file length does not match the expected length. $actual_len != $expected_len" + fi + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav + fi +} + +TOOL_CHK_LOGLEN() { + expect=$1 + shift + + echo "running tool_chk_loglen" + + # Run test. + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + ) + + expect_len="`wc -l < $expect`" + if [ "$expect_len" -gt 0 ]; then + echo " PASSED" + else + echo "*FAILED*" + echo " The generated .log file is empty!." + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $expect + fi +} + + +# Print a "SKIP" message +SKIP() { + TESTING $H5DWALK $@6 + echo " -SKIP-" +} + + + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +TOOLTEST help-1.txt -h +TOOLTEST help-2.txt --help +TOOL_LOGTEST h5diff_basic1.h5_h5dump -l -T $H5DUMP_BIN -n ./h5diff_basic1.h5 +TOOL_CHK_LOGLEN showme-h5dump.log -o `pwd`/showme-h5dump.log -T $H5DUMP_BIN -n `pwd` + + +# +# +# Clean up temporary files/directories +CLEAN_TESTFILES_AND_TESTDIR + +if test $nerrors -eq 0 ; then + echo "All $TESTNAME tests passed." + exit $EXIT_SUCCESS +else + echo "$TESTNAME tests failed with $nerrors errors." + exit $EXIT_FAILURE +fi + |