[svn-r5554] Purpose:

    Compress I/O Test Add
Description:
    This is the initial check-in of the compress I/O test. It will write
    out compressed buffers to a file.

    I need to implement a write of uncompressed data first for
    comparison...
Platforms tested:
    Linux

3 files changed, 487 insertions, 2 deletions

diff --git a/MANIFEST b/MANIFEST
index 4c5aadf..dbb3656 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -756,6 +756,7 @@
 ./perform/pio_perf.h
 ./perform/pio_timer.c
 ./perform/pio_timer.h
+./perform/zip_perf.c
 
 ./release_docs/INSTALL
 ./release_docs/INSTALL_TFLOPS
diff --git a/perform/Makefile.in b/perform/Makefile.in
index 46e7621..09f9086 100644
--- a/perform/Makefile.in
+++ b/perform/Makefile.in
@@ -20,7 +20,7 @@ LIBTOOLS=../tools/lib/libh5tools.la
 ## These are the programs that `make all' or `make tests' will build and which
 ## `make check' will run. List them in the order they should be run.
 TEST_PROGS_PARA=pio_perf
-TEST_PROGS=iopipe chunk overhead
+TEST_PROGS=iopipe chunk overhead zip_perf
 
 ## These are the files that `make clean' (and derivatives) will remove from
 ## this directory.
@@ -34,7 +34,7 @@ PIO_PERF_OBJ=$(PIO_PERF_SRC:.c=.lo)
 
 TEST_SRC_PARA=$(PIO_PERF_SRC)
 
-TEST_SRC=iopipe.c chunk.c overhead.c $(TEST_SRC_PARA)
+TEST_SRC=iopipe.c chunk.c overhead.c zip_perf.c $(TEST_SRC_PARA)
 TEST_OBJ=$(TEST_SRC:.c=.lo)
 
 ## How to build the programs... they all depend on the hdf5 library
@@ -61,4 +61,8 @@ chunk: chunk.lo
 overhead: overhead.lo
 	@$(LT_LINK_EXE) $(CFLAGS) -o $@ overhead.lo $(LIB) $(LIBHDF5) $(LDFLAGS) $(LIBS)
 
+zip_perf: zip_perf.lo
+	@$(LT_LINK_EXE) $(CFLAGS) -o $@ zip_perf.lo $(LIBH5TEST) \
+	$(LIBTOOLS) $(LIBHDF5) $(LDFLAGS) $(LIBS)
+
 @CONCLUDE@
diff --git a/perform/zip_perf.c b/perform/zip_perf.c
new file mode 100644
index 0000000..066fce9
--- /dev/null
+++ b/perform/zip_perf.c
@@ -0,0 +1,480 @@
+/*
+ * Copyright (C) 2002
+ *     National Center for Supercomputing Applications
+ *     All rights reserved.
+ */
+
+/* @(#) $Id$ */
+
+/* ===========================================================================
+ * Usage:  zip_perf [-d] [-f] [-h] [-1 to -9] [files...]
+ *   -d : decompress
+ *   -f : compress with Z_FILTERED
+ *   -h : compress with Z_HUFFMAN_ONLY
+ *   -1 to -9 : compression level
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/time.h>
+
+#include "zlib.h"
+
+/* our header files */
+#include "hdf5.h"
+#include "h5tools_utils.h"
+
+#if defined(MSDOS) || defined(OS2) || defined(WIN32)
+#  include <fcntl.h>
+#  include <io.h>
+#  define SET_BINARY_MODE(file)     setmode(fileno(file), O_BINARY)
+#else
+#  define SET_BINARY_MODE(file)     /* nothing */
+#endif  /* MSDOS || OS2 || WIN32 */
+
+#ifdef VMS
+#  define unlink        delete
+#  define GZ_SUFFIX     "-gz"
+#endif  /* VMS */
+
+#ifdef RISCOS
+#  define unlink        remove
+#  define GZ_SUFFIX     "-gz"
+#  define fileno(file)  file->__file
+#endif  /* RISCOS */
+
+#if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os
+#  include <unix.h>     /* for fileno */
+#endif  /* __MWERKS__ ... */
+
+#ifndef GZ_SUFFIX
+#  define GZ_SUFFIX     ".gz"
+#endif  /* GZ_SUFFIX */
+
+#define SUFFIX_LEN          (sizeof(GZ_SUFFIX) - 1)
+
+#define ONE_KB              1024
+#define ONE_MB              (ONE_KB * ONE_KB)
+
+#define MICROSECOND         1000000.0
+
+/* report 0.0 in case t is zero too */
+#define MB_PER_SEC(bytes,t) (((t)==0.0) ? 0.0 : ((((double)bytes) / ONE_MB) / (t)))
+
+/* report 0.0 in case t is zero too */
+#define MB_PER_SEC(bytes,t) (((t)==0.0) ? 0.0 : ((((double)bytes) / ONE_MB) / (t)))
+
+#ifndef TRUE
+#define TRUE    1
+#endif  /* TRUE */
+
+#ifndef FALSE
+#define FALSE   (!TRUE)
+#endif  /* FALSE */
+
+#define BUFLEN              (16 * ONE_KB)
+#define MAX_NAME_LEN        ONE_KB
+
+/* internal variables */
+static const char *prog;
+static const char *option_prefix;
+static char *filename;
+static int compress_level = Z_DEFAULT_COMPRESSION;
+static int output;
+
+/* internal functions */
+static void error(const char *fmt, ...);
+static void compress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source,
+                            uLong sourceLen);
+
+/* commandline options : long and short form */
+static const char *s_opts = "hB:b:p:s:0123456789";
+static struct long_options l_opts[] = {
+    { "help", no_arg, 'h' },
+    { "file-size", require_arg, 's' },
+    { "file-siz", require_arg, 's' },
+    { "file-si", require_arg, 's' },
+    { "file-s", require_arg, 's' },
+    { "file", require_arg, 's' },
+    { "fil", require_arg, 's' },
+    { "fi", require_arg, 's' },
+    { "max-buffer-size", require_arg, 'B' },
+    { "max-buffer-siz", require_arg, 'B' },
+    { "max-buffer-si", require_arg, 'B' },
+    { "max-buffer-s", require_arg, 'B' },
+    { "max-buffer", require_arg, 'B' },
+    { "max-buffe", require_arg, 'B' },
+    { "max-buff", require_arg, 'B' },
+    { "max-buf", require_arg, 'B' },
+    { "max-bu", require_arg, 'B' },
+    { "max-b", require_arg, 'B' },
+    { "max", require_arg, 'B' },
+    { "min-buffer-size", require_arg, 'b' },
+    { "min-buffer-siz", require_arg, 'b' },
+    { "min-buffer-si", require_arg, 'b' },
+    { "min-buffer-s", require_arg, 'b' },
+    { "min-buffer", require_arg, 'b' },
+    { "min-buffe", require_arg, 'b' },
+    { "min-buff", require_arg, 'b' },
+    { "min-buf", require_arg, 'b' },
+    { "min-bu", require_arg, 'b' },
+    { "min-b", require_arg, 'b' },
+    { "min", require_arg, 'b' },
+    { "prefix", require_arg, 'p' },
+    { "prefi", require_arg, 'p' },
+    { "pref", require_arg, 'p' },
+    { "pre", require_arg, 'p' },
+    { "pr", require_arg, 'p' },
+    { NULL, 0, '\0' }
+};
+
+/*
+ * Function:    error
+ * Purpose:     Display error message and exit.
+ * Programmer:  Bill Wendling, 05. June 2002
+ * Modifications:
+ */
+static void
+error(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    fprintf(stderr, "%s: error: ", prog);
+    vfprintf(stderr, fmt, ap);
+    fprintf(stderr, "\n");
+    va_end(ap);
+    exit(EXIT_FAILURE);
+}
+
+/*
+ * Function:    cleanup
+ * Purpose:     Cleanup the output file.
+ * Returns:     Nothing
+ * Programmer:  Bill Wendling, 06. June 2002
+ * Modifications:
+ */
+static void
+cleanup(void)
+{
+    if (!getenv("HDF5_NOCLEANUP"))
+        unlink(filename);
+}
+
+static void
+write_file(Bytef *source, uLongf sourceLen)
+{
+    Bytef *d_ptr, *dest;
+    uLongf d_len, destLen;
+
+    /* destination buffer needs to be at least 0.1% larger than sourceLen
+     * plus 12 bytes */
+    destLen = (uLongf)((double)sourceLen + (sourceLen * 0.1)) + 12;
+    dest = (Bytef *)malloc(destLen);
+
+    if (!dest)
+        error("out of memory");
+
+    compress_buffer(dest, &destLen, source, sourceLen);
+
+    d_ptr = dest;
+    d_len = destLen;
+
+    /* loop to make sure we write everything out that we want to write */
+    for (;;) {
+        int rc = write(output, d_ptr, d_len);
+
+        if (rc == -1)
+            error(strerror(errno));
+
+        if (rc == d_len)
+            break;
+
+        d_len -= rc;
+        d_ptr += rc;
+    }
+}
+
+/*
+ * Function:    compress_buffer
+ * Purpose:     Compress the buffer.
+ * Returns:     Z_OK            - success
+ *              Z_MEM_ERROR     - not enough memory
+ *              Z_BUF_ERROR     - not enough room in the output buffer
+ *              Z_STREAM_ERROR  - level parameter is invalid
+ * Programmer:  Bill Wendling, 05. June 2002
+ * Modifications:
+ */
+static void
+compress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source,
+                uLong sourceLen)
+{
+    int rc = compress2(dest, destLen, source, sourceLen, compress_level);
+
+    if (rc != Z_OK) {
+        /* compress2 failed - cleanup and tell why */
+        cleanup();
+
+        switch (rc) {
+        case Z_MEM_ERROR:
+            error("not enough memory");
+            break;
+        case Z_BUF_ERROR:
+            error("not enough room in the output buffer");
+            break;
+        case Z_STREAM_ERROR:
+            error("level parameter (%d) is invalid", compress_level);
+            break;
+        default:
+            error("unknown compression error");
+            break;
+        }
+    }
+}
+
+/*
+ * Function:    uncompress_buffer
+ * Purpose:     Uncompress the buffer.
+ * Returns:     Z_OK            - success
+ *              Z_MEM_ERROR     - not enough memory
+ *              Z_BUF_ERROR     - not enough room in the output buffer
+ *              Z_DATA_ERROR    - the input data was corrupted
+ * Programmer:  Bill Wendling, 05. June 2002
+ * Modifications:
+ */
+static int
+uncompress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source,
+                  uLong sourceLen)
+{
+    int rc = uncompress(dest, destLen, source, sourceLen);
+
+    return rc;
+}
+
+/*
+ * Function:    get_unique_name
+ * Purpose:     Create a new file who's name doesn't conflict with
+ *              pre-existing files.
+ * Returns:
+ * Programmer:  Bill Wendling, 06. June 2002
+ * Modifications:
+ */
+static void
+get_unique_name(void)
+{
+    const char *prefix = "/tmp", *template = "/zip_perf_XXXXXX";
+    const char *env = getenv("HDF5_PREFIX");
+
+    if (env)
+        prefix = env;
+
+    if (option_prefix)
+        prefix = option_prefix;
+
+    filename = calloc(1, strlen(prefix) + strlen(template) + 1);
+
+    if (!filename)
+        error("out of memory");
+
+    strcpy(filename, prefix);
+    strcat(filename, template);
+    output = mkstemp(filename);
+
+    if (output == -1)
+        error(strerror(errno));
+}
+
+/*
+ * Function:    usage
+ * Purpose:     Print a usage message and then exit.
+ * Return:      Nothing
+ * Programmer:  Bill Wendling, 05. June 2002
+ * Modifications:
+ */
+static void
+usage(void)
+{
+    printf("usage: %s [OPTIONS]\n", prog);
+    printf("  OPTIONS\n");
+    printf("     -h, --help                 Print this usage message and exit\n");
+    printf("     -1...-9                    Level of compression, from 1 to 9\n");
+    printf("     -s S, --file-size=S        Maximum size of uncompressed file [default: 64M]\n");
+    printf("     -B S, --max-buffer_size=S  Maximum size of buffer [default: 1M]\n");
+    printf("     -b S, --min-buffer_size=S  Minumum size of buffer [default: 128K]\n");
+    printf("     -p D, --prefix=D           The directory prefix to place the file\n");
+    printf("                                [default: /tmp]\n");
+    printf("\n");
+    printf("  D  - a directory which exists\n");
+    printf("  S  - is a size specifier, an integer >=0 followed by a size indicator:\n");
+    printf("\n");
+    printf("          K - Kilobyte (%d)\n", ONE_KB);
+    printf("          M - Megabyte (%d)\n", ONE_MB);
+    printf("\n");
+    printf("      Example: 37M = 37 Megabytes = %d bytes\n", 37 * ONE_MB);
+    printf("\n");
+    fflush(stdout);
+}
+
+/*
+ * Function:    parse_size_directive
+ * Purpose:     Parse the size directive passed on the commandline. The size
+ *              directive is an integer followed by a size indicator:
+ *
+ *                  K, k - Kilobyte
+ *                  M, m - Megabyte
+ *
+ * Return:      The size as a size_t because this is related to buffer size.
+ *              If an unknown size indicator is used, then the program will
+ *              exit with EXIT_FAILURE as the return value.
+ * Programmer:  Bill Wendling, 05. June 2002
+ * Modifications:
+ */
+static unsigned long
+parse_size_directive(const char *size)
+{
+    unsigned long s;
+    char *endptr;
+
+    s = strtoul(size, &endptr, 10);
+
+    if (endptr && *endptr) {
+        while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t'))
+            ++endptr;
+
+        switch (*endptr) {
+            case 'K':
+            case 'k':
+                s *= ONE_KB;
+                break;
+            case 'M':
+            case 'm':
+                s *= ONE_MB;
+                break;
+            default:
+                error("illegal size specifier '%c'", *endptr);
+                break;
+        }
+    }
+
+    return s;
+}
+
+static void
+do_write_test(unsigned long file_size, unsigned long min_buf_size,
+              unsigned long max_buf_size)
+{
+    uLongf src_len, total_len;
+    struct timeval timer_start, timer_stop;
+    double total_time;
+    Bytef *src;
+
+    for (src_len = min_buf_size; src_len <= max_buf_size; src_len <<= 1) {
+        src = (Bytef *)calloc(1, sizeof(Bytef) * src_len);
+
+        if (!src) {
+            cleanup();
+            error("out of memory");
+        }
+
+        printf("Buffer size == ");
+
+        if (src_len >= ONE_KB && (src_len % ONE_KB) == 0) {
+            if (src_len >= ONE_MB && (src_len % ONE_MB) == 0) {
+                printf("%ldMB", src_len / ONE_MB);
+            } else {
+                printf("%ldKB", src_len / ONE_KB);
+            }
+        } else {
+            printf("%ld", src_len);
+        }
+
+        printf("\n");
+        gettimeofday(&timer_start, NULL);
+        output = open(filename, O_RDWR | O_TRUNC);
+
+        if (output == -1)
+            error(strerror(errno));
+
+        for (total_len = 0; total_len < file_size; total_len += src_len) {
+            write_file(src, src_len);
+        }
+
+        close(output);
+        gettimeofday(&timer_stop, NULL);
+
+        total_time = ((double)timer_stop.tv_sec +
+                            ((double)timer_stop.tv_usec) / MICROSECOND) -
+                     ((double)timer_start.tv_sec +
+                            ((double)timer_start.tv_usec) / MICROSECOND);
+
+        printf("\tCompressed Write Time: %.2fs\n", total_time);
+        printf("\tCompressed Write Throughput: %.2fMB/s\n",
+               MB_PER_SEC(file_size, total_time));
+
+        free(src);
+    }
+}
+
+/*
+ * Function:    main
+ * Purpose:     Run the program
+ * Return:      EXIT_SUCCESS or EXIT_FAILURE
+ * Programmer:  Bill Wendling, 05. June 2002
+ * Modifications:
+ */
+int
+main(int argc, char **argv)
+{
+    unsigned long min_buf_size = 128 * ONE_KB, max_buf_size = ONE_MB;
+    unsigned long file_size = 64 * ONE_MB;
+    int opt;
+
+    prog = argv[0];
+
+    while ((opt = get_option(argc, (const char **)argv, s_opts, l_opts)) > 0) {
+        switch ((char)opt) {
+        case '0': case '1': case '2':
+        case '3': case '4': case '5':
+        case '6': case '7': case '8':
+        case '9':
+            compress_level = opt - '0';
+            break;
+        case 'B':
+            max_buf_size = parse_size_directive(opt_arg);
+            break;
+        case 'b':
+            min_buf_size = parse_size_directive(opt_arg);
+            break;
+        case 'p':
+            option_prefix = opt_arg;
+            break;
+        case 's':
+            file_size = parse_size_directive(opt_arg);
+            break;
+        case '?':
+            usage();
+            exit(EXIT_FAILURE);
+            break;
+        case 'h':
+        default:
+            usage();
+            exit(EXIT_SUCCESS);
+            break;
+        }
+    }
+
+    if (min_buf_size > max_buf_size)
+        error("minmum buffer size (%d) exceeds maximum buffer size (%d)",
+              min_buf_size, max_buf_size);
+
+    get_unique_name();
+    do_write_test(file_size, min_buf_size, max_buf_size);
+    cleanup();
+    return EXIT_SUCCESS;
+}