diff options
-rw-r--r-- | test/swmr_addrem_writer.c | 101 | ||||
-rw-r--r-- | test/swmr_common.c | 350 | ||||
-rw-r--r-- | test/swmr_common.h | 135 | ||||
-rw-r--r-- | test/swmr_generator.c | 612 | ||||
-rw-r--r-- | test/swmr_reader.c | 881 | ||||
-rw-r--r-- | test/swmr_remove_reader.c | 159 | ||||
-rw-r--r-- | test/swmr_remove_writer.c | 100 | ||||
-rw-r--r-- | test/swmr_sparse_reader.c | 125 | ||||
-rw-r--r-- | test/swmr_sparse_writer.c | 97 | ||||
-rw-r--r-- | test/swmr_writer.c | 752 | ||||
-rwxr-xr-x | test/testswmr.sh | 2 |
11 files changed, 2124 insertions, 1190 deletions
diff --git a/test/swmr_addrem_writer.c b/test/swmr_addrem_writer.c index b13ff3e..e80cea9 100644 --- a/test/swmr_addrem_writer.c +++ b/test/swmr_addrem_writer.c @@ -13,12 +13,67 @@ * access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*-------------------------------------------------------------------------
+ *
+ * Created: swmr_addrem_writer.c
+ *
+ * Purpose: Adds and removes data to a randomly selected subset of the
+ * datasets in the SWMR test file.
+ *
+ * This program is intended to run concurrently with the
+ * swmr_reader program. It is also run AFTER a sequential
+ * (not concurrent!) invoking of swmr_writer so the writer
+ * can dump a bunch of data into the datasets. Otherwise,
+ * there wouldn't be much to shrink :)
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include <assert.h>
#include <sys/time.h>
#include "swmr_common.h"
-#define MAX_SIZE_CHANGE 10
+/****************/
+/* Local Macros */
+/****************/
+/* The maximum # of records to add/remove from the dataset in one step */
+#define MAX_SIZE_CHANGE 10
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static hid_t
+open_skeleton(const char *filename, unsigned verbose);
+static int addrem_records(hid_t fid, unsigned verbose, unsigned long nops,
+ unsigned long flush_count);
+static void usage(void);
+
+
+/*-------------------------------------------------------------------------
+ * Function: open_skeleton
+ *
+ * Purpose: Opens the SWMR HDF5 file and datasets.
+ *
+ * Parameters: const char *filename
+ * The filename of the SWMR HDF5 file to open
+ *
+ * unsigned verbose
+ * Whether or not to emit verbose console messages
+ *
+ * Return: Success: The file ID of the opened SWMR file
+ * The dataset IDs are stored in a global array
+ *
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static hid_t
open_skeleton(const char *filename, unsigned verbose)
{
@@ -28,6 +83,8 @@ open_skeleton(const char *filename, unsigned verbose) hsize_t dim[2]; /* Dataspace dimension */
unsigned u, v; /* Local index variable */
+ assert(filename);
+
/* Create file access property list */
if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
return -1;
@@ -81,6 +138,30 @@ open_skeleton(const char *filename, unsigned verbose) return(fid);
}
+
+/*-------------------------------------------------------------------------
+ * Function: addrem_records
+ *
+ * Purpose: Adds/removes a specified number of records to random datasets
+ * to the SWMR test file.
+ *
+ * Parameters: hid_t fid
+ * The file ID of the SWMR HDF5 file
+ *
+ * unsigned verbose
+ * Whether or not to emit verbose console messages
+ *
+ * unsigned long nops
+ * # of records to read/write in the datasets
+ *
+ * unsigned long flush_count
+ * # of records to write before flushing the file to disk
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static int
addrem_records(hid_t fid, unsigned verbose, unsigned long nops, unsigned long flush_count)
{
@@ -94,6 +175,8 @@ addrem_records(hid_t fid, unsigned verbose, unsigned long nops, unsigned long fl unsigned long op_to_flush; /* # of operations before flush */
unsigned long u, v; /* Local index variables */
+ assert(fid > 0);
+
/* Reset the buffer */
memset(&buf, 0, sizeof(buf));
@@ -219,10 +302,20 @@ addrem_records(hid_t fid, unsigned verbose, unsigned long nops, unsigned long fl static void
usage(void)
{
+ printf("\n");
printf("Usage error!\n");
- printf("Usage: swmr_addrem_writer [-q] [-f <# of operations between flushing file contents>] [-r <random # seed>] <# of shrinks>\n");
- printf("<# of operations between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of shrinks> - 1)\n");
- printf("Defaults to verbose (no '-q' given) and flushing every 1000 operations('-f 1000')\n");
+ printf("\n");
+ printf("Usage: swmr_addrem_writer [-q] [-f <# of operations between flushing\n");
+ printf(" file contents>] [-r <random seed>] <# of operations>\n");
+ printf("\n");
+ printf("<# of operations between flushing file contents> should be 0 (for\n");
+ printf("no flushing) or between 1 and (<# of operations> - 1).\n");
+ printf("\n");
+ printf("<# of operations> must be specified.\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given), flushing every 1000 operations\n");
+ printf("('-f 1000'), and will generate a random seed (no -r given).\n");
+ printf("\n");
exit(1);
}
diff --git a/test/swmr_common.c b/test/swmr_common.c index f857263..b921d15 100644 --- a/test/swmr_common.c +++ b/test/swmr_common.c @@ -1,108 +1,242 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * Copyright by The HDF Group. *
- * Copyright by the Board of Trustees of the University of Illinois. *
- * All rights reserved. *
- * *
- * This file is part of HDF5. The full HDF5 copyright notice, including *
- * terms governing use, modification, and redistribution, is contained in *
- * the files COPYING and Copyright.html. COPYING can be found at the root *
- * of the source code distribution tree; Copyright.html can be found at the *
- * root level of an installed copy of the electronic HDF5 document set and *
- * is linked from the top-level documents page. It can also be found at *
- * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
- * access to either file, you may request a copy from help@hdfgroup.org. *
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-#include "swmr_common.h"
-
-static unsigned symbol_mapping[NMAPPING] = {0, 0, 0, 0, 1, 1, 2, 3, 4};
-unsigned symbol_count[NLEVELS] = {100, 200, 400, 800, 1600};
-
-symbol_info_t *symbol_info[NLEVELS];
-
-symbol_info_t *
-choose_dataset(void)
-{
- unsigned level; /* The level of the dataset */
- unsigned offset; /* The "offset" of the dataset at that level */
-
- /* Determine level of dataset */
- level = symbol_mapping[random() % NMAPPING];
-
- /* Determine the offset of the level */
- offset = random() % symbol_count[level];
-
- return(&symbol_info[level][offset]);
-} /* end choose_dataset() */
-
-hid_t
-create_symbol_datatype(void)
-{
- hid_t sym_type_id; /* Datatype ID for symbol */
- hid_t opaq_type_id; /* Datatype ID for opaque part of record */
-
- /* Create opaque datatype to represent other information for this record */
- if((opaq_type_id = H5Tcreate(H5T_OPAQUE, DTYPE_SIZE)) < 0)
- return -1;
-
- /* Create compound datatype for symbol */
- if((sym_type_id = H5Tcreate(H5T_COMPOUND, sizeof(symbol_t))) < 0)
- return -1;
-
- /* Insert fields in symbol datatype */
- if(H5Tinsert(sym_type_id, "rec_id", HOFFSET(symbol_t, rec_id), H5T_NATIVE_UINT64) < 0)
- return -1;
- if(H5Tinsert(sym_type_id, "info", HOFFSET(symbol_t, info), opaq_type_id) < 0)
- return -1;
-
- /* Close opaque datatype */
- if(H5Tclose(opaq_type_id) < 0)
- return -1;
-
- return(sym_type_id);
-} /* end create_symbol_datatype() */
-
-int
-generate_name(char *name_buf, unsigned level, unsigned count)
-{
- sprintf(name_buf, "%u-%04u", level, count);
-
- return 0;
-} /* end generate_name() */
-
-int
-generate_symbols(void)
-{
- unsigned u, v; /* Local index variables */
-
- for(u = 0; u < NLEVELS; u++) {
- symbol_info[u] = (symbol_info_t *)malloc(symbol_count[u] * sizeof(symbol_info_t));
- for(v = 0; v < symbol_count[u]; v++) {
- char name_buf[64];
-
- generate_name(name_buf, u, v);
- symbol_info[u][v].name = (char *)malloc(strlen(name_buf) + 1);
- strcpy(symbol_info[u][v].name, name_buf);
- symbol_info[u][v].dsid = -1;
- symbol_info[u][v].nrecords = 0;
- } /* end for */
- } /* end for */
-
- return 0;
-} /* end generate_symbols() */
-
-int
-shutdown_symbols(void)
-{
- unsigned u, v; /* Local index variables */
-
- /* Clean up the symbols */
- for(u = 0; u < NLEVELS; u++) {
- for(v = 0; v < symbol_count[u]; v++)
- free(symbol_info[u][v].name);
- free(symbol_info[u]);
- } /* end for */
-
- return 0;
-} /* end shutdown_symbols() */
-
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: swmr_common.c + * + * Purpose: Utility functions for the SWMR test code. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <assert.h> + +#include "swmr_common.h" + +/*******************/ +/* Local Variables */ +/*******************/ + +/* The SWMR data arrays: + * + * The code uses a 2-D jagged array of datasets. The first dimension is called + * the 'level' and there are five of them. + * + * #define NLEVELS 5 + * + * The second dimension is the 'count' and there are quite a few datasets per + * 'level'. + * + * unsigned symbol_count[NLEVELS] = {100, 200, 400, 800, 1600}; + * + * These datasets are created when the skeleton is generated and are initially + * empty. Each dataset has no upper bound on size (H5S_UNLIMITED). They + * are of compound type, with two members: an integer ID and an opaque + * 'data part'. The data part is not used by the SWMR testing. + * + * The SWMR testing will then randomly add and/or remove entries + * from these datasets. The selection of the level is skewed by a mapping + * table which preferentially hammers on the lower levels with their smaller + * number of datasets. + * + * static unsigned symbol_mapping[NMAPPING] = {0, 0, 0, 0, 1, 1, 2, 3, 4}; + * + * The information about each dataset (name, hid_t, etc.) is stored in a + * separate array. + * + * symbol_info_t *symbol_info[NLEVELS]; + */ + +/* An array of dataset levels, used to select the level for a SWMR operation + * Note that this preferentially selects the lower levels with their smaller + * number of datasets. + */ +static unsigned symbol_mapping[NMAPPING] = {0, 0, 0, 0, 1, 1, 2, 3, 4}; + +/* The number of datasets at each level */ +unsigned symbol_count[NLEVELS] = {100, 200, 400, 800, 1600}; + +/* Array of dataset information entries (1 per dataset) */ +symbol_info_t *symbol_info[NLEVELS]; + + +/*------------------------------------------------------------------------- + * Function: choose_dataset + * + * Purpose: Selects a random dataset in the SWMR file + * + * Parameters: N/A + * + * Return: Success: A pointer to information about a dataset. + * Failure: Can't fail + * + *------------------------------------------------------------------------- + */ +symbol_info_t * +choose_dataset(void) +{ + unsigned level; /* The level of the dataset */ + unsigned offset; /* The "offset" of the dataset at that level */ + + /* Determine level of dataset */ + level = symbol_mapping[random() % NMAPPING]; + + /* Determine the offset of the level */ + offset = random() % symbol_count[level]; + + return(&symbol_info[level][offset]); +} /* end choose_dataset() */ + + +/*------------------------------------------------------------------------- + * Function: create_symbol_datatype + * + * Purpose: Create's the HDF5 datatype used for elements in the SWMR + * testing datasets. + * + * Parameters: N/A + * + * Return: Success: An HDF5 type ID + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +hid_t +create_symbol_datatype(void) +{ + hid_t sym_type_id; /* Datatype ID for symbol */ + hid_t opaq_type_id; /* Datatype ID for opaque part of record */ + + /* Create opaque datatype to represent other information for this record */ + if((opaq_type_id = H5Tcreate(H5T_OPAQUE, DTYPE_SIZE)) < 0) + return -1; + + /* Create compound datatype for symbol */ + if((sym_type_id = H5Tcreate(H5T_COMPOUND, sizeof(symbol_t))) < 0) + return -1; + + /* Insert fields in symbol datatype */ + if(H5Tinsert(sym_type_id, "rec_id", HOFFSET(symbol_t, rec_id), H5T_NATIVE_UINT64) < 0) + return -1; + if(H5Tinsert(sym_type_id, "info", HOFFSET(symbol_t, info), opaq_type_id) < 0) + return -1; + + /* Close opaque datatype */ + if(H5Tclose(opaq_type_id) < 0) + return -1; + + return(sym_type_id); +} /* end create_symbol_datatype() */ + + +/*------------------------------------------------------------------------- + * Function: generate_name + * + * Purpose: Generates a SWMR testing dataset name given a level and + * count. + * The name is in the format <name>-<level> (%u-%04u). + * + * Parameters: char *name_buf + * Buffer for the created name. Must be pre-allocated. + * Since the name is formulaic, this isn't considered an issue. + * + * unsigned level + * The dataset's level + * + * unsigned count + * The dataset's count + * + * Return: Success: 0 + * + * Failure: Can't fail + * + *------------------------------------------------------------------------- + */ +int +generate_name(char *name_buf, unsigned level, unsigned count) +{ + assert(name_buf); + + sprintf(name_buf, "%u-%04u", level, count); + + return 0; +} /* end generate_name() */ + + +/*------------------------------------------------------------------------- + * Function: generate_symbols + * + * Purpose: Initializes the global dataset infomration arrays. + * + * Parameters: N/A + * + * Return: Success: 0 + * Failure: Can't fail + * + *------------------------------------------------------------------------- + */ +int +generate_symbols(void) +{ + unsigned u, v; /* Local index variables */ + + for(u = 0; u < NLEVELS; u++) { + symbol_info[u] = (symbol_info_t *)malloc(symbol_count[u] * sizeof(symbol_info_t)); + for(v = 0; v < symbol_count[u]; v++) { + char name_buf[64]; + + generate_name(name_buf, u, v); + symbol_info[u][v].name = (char *)malloc(strlen(name_buf) + 1); + strcpy(symbol_info[u][v].name, name_buf); + symbol_info[u][v].dsid = -1; + symbol_info[u][v].nrecords = 0; + } /* end for */ + } /* end for */ + + return 0; +} /* end generate_symbols() */ + + +/*------------------------------------------------------------------------- + * Function: shutdown_symbols + * + * Purpose: Cleans up the global dataset information arrays. + * + * Parameters: N/A + * + * Return: Success: 0 + * Failure: Can't fail + * + *------------------------------------------------------------------------- + */ +int +shutdown_symbols(void) +{ + unsigned u, v; /* Local index variables */ + + /* Clean up the symbols */ + for(u = 0; u < NLEVELS; u++) { + for(v = 0; v < symbol_count[u]; v++) + free(symbol_info[u][v].name); + free(symbol_info[u]); + } /* end for */ + + return 0; +} /* end shutdown_symbols() */ diff --git a/test/swmr_common.h b/test/swmr_common.h index edf0f3b..83491ee 100644 --- a/test/swmr_common.h +++ b/test/swmr_common.h @@ -1,56 +1,79 @@ -#ifndef __BENCHMARK_COMMON_H
-#define __BENCHMARK_COMMON_H
-
-/* Headers needed */
-
-#define _GNU_SOURCE
-#include <limits.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <time.h>
-#include "hdf5.h"
-
-/* Macros */
-
-#ifndef TRUE
-#define TRUE 1
-#endif /* TRUE */
-#ifndef FALSE
-#define FALSE 0
-#endif /* FALSE */
-
-#define NLEVELS 5
-#define NMAPPING 9
-
-#define FILENAME "swmr_data.h5"
-#define DTYPE_SIZE 150
-
-/* Typedefs */
-
-/* Information about a symbol/dataset */
-typedef struct {
- char *name; /* Dataset name for symbol */
- hid_t dsid; /* Dataset ID for symbol */
- hsize_t nrecords; /* Number of records for the symbol */
-} symbol_info_t;
-
-/* A symbol's record */
-typedef struct {
- uint64_t rec_id; /* ID for this record (unique in symbol) */
- uint8_t info[DTYPE_SIZE]; /* "Other" information for this record */
-} symbol_t;
-
-/* Global variables */
-extern symbol_info_t *symbol_info[NLEVELS];
-extern unsigned symbol_count[NLEVELS];
-
-/* Prototypes */
-extern symbol_info_t * choose_dataset(void);
-extern hid_t create_symbol_datatype(void);
-extern int generate_name(char *name_buf, unsigned level, unsigned count);
-extern int generate_symbols(void);
-extern int shutdown_symbols(void);
-
-#endif /* __BENCHMARK_COMMON_H */
-
+#ifndef _SWMR_COMMON_H +#define _SWMR_COMMON_H + +/* Headers needed */ + +#define _GNU_SOURCE +#include <limits.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <time.h> +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "hdf5.h" + +/**********/ +/* Macros */ +/**********/ + +#ifndef TRUE +#define TRUE 1 +#endif /* TRUE */ +#ifndef FALSE +#define FALSE 0 +#endif /* FALSE */ + +#define NLEVELS 5 /* # of datasets in the SWMR test file */ + +#define NMAPPING 9 + +#define FILENAME "swmr_data.h5" /* SWMR test file name */ +#define DTYPE_SIZE 150 /* Data size in opaque type */ + +/************/ +/* Typedefs */ +/************/ + +/* Information about a symbol/dataset */ +typedef struct { + char *name; /* Dataset name for symbol */ + hid_t dsid; /* Dataset ID for symbol */ + hsize_t nrecords; /* Number of records for the symbol */ +} symbol_info_t; + +/* A symbol's record */ +typedef struct { + uint64_t rec_id; /* ID for this record (unique in symbol) */ + uint8_t info[DTYPE_SIZE]; /* "Other" information for this record */ +} symbol_t; + +/********************/ +/* Global Variables */ +/********************/ +extern symbol_info_t *symbol_info[NLEVELS]; +extern unsigned symbol_count[NLEVELS]; + +/**************/ +/* Prototypes */ +/**************/ +symbol_info_t * choose_dataset(void); +hid_t create_symbol_datatype(void); +int generate_name(char *name_buf, unsigned level, unsigned count); +int generate_symbols(void); +int shutdown_symbols(void); + +#endif /* _SWMR_COMMON_H */ diff --git a/test/swmr_generator.c b/test/swmr_generator.c index 1da536a..f904378 100644 --- a/test/swmr_generator.c +++ b/test/swmr_generator.c @@ -1,272 +1,340 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * Copyright by The HDF Group. *
- * Copyright by the Board of Trustees of the University of Illinois. *
- * All rights reserved. *
- * *
- * This file is part of HDF5. The full HDF5 copyright notice, including *
- * terms governing use, modification, and redistribution, is contained in *
- * the files COPYING and Copyright.html. COPYING can be found at the root *
- * of the source code distribution tree; Copyright.html can be found at the *
- * root level of an installed copy of the electronic HDF5 document set and *
- * is linked from the top-level documents page. It can also be found at *
- * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
- * access to either file, you may request a copy from help@hdfgroup.org. *
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-#include <sys/time.h>
-
-#include "swmr_common.h"
-
-#define CHUNK_SIZE 50
-
-static int
-gen_skeleton(const char *filename, unsigned verbose, int comp_level,
- const char *index_type, unsigned random_seed)
-{
- hid_t fid; /* File ID for new HDF5 file */
- hid_t fcpl; /* File creation property list */
- hid_t fapl; /* File access property list */
- hid_t dcpl; /* Dataset creation property list */
- hid_t tid; /* Datatype for dataset elements */
- hid_t sid; /* Dataspace ID */
- hid_t aid; /* Attribute ID */
- hsize_t dims[2] = {1, 0}; /* Dataset starting dimensions */
- hsize_t max_dims[2] = {1, H5S_UNLIMITED}; /* Dataset maximum dimensions */
- hsize_t chunk_dims[2] = {1, CHUNK_SIZE}; /* Chunk dimensions */
-#ifdef FILLVAL_WORKS
- symbol_t fillval; /* Dataset fill value */
-#endif /* FILLVAL_WORKS */
- unsigned u, v; /* Local index variable */
-
- /* Create file access property list */
- if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
- return -1;
-
- /* Select the correct index type */
- if(strcmp(index_type, "b1"))
- if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
- return -1;
- if(!strcmp(index_type, "b2"))
- max_dims[0] = H5S_UNLIMITED;
-
-#ifdef QAK
- /* Increase the initial size of the metadata cache */
- {
- H5AC_cache_config_t mdc_config;
-
- mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
- H5Pget_mdc_config(fapl, &mdc_config);
- fprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size);
- fprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length);
- mdc_config.set_initial_size = 1;
- mdc_config.initial_size = 16 * 1024 * 1024;
- /* mdc_config.epoch_length = 5000; */
- H5Pset_mdc_config(fapl, &mdc_config);
- }
-#endif /* QAK */
-
-#ifdef QAK
- H5Pset_small_data_block_size(fapl, (hsize_t)(50 * CHUNK_SIZE * DTYPE_SIZE));
-#endif /* QAK */
-
-#ifdef QAK
- H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
-#endif /* QAK */
-
- /* Create file creation property list */
- if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
- return -1;
-
-#ifdef QAK
- H5Pset_link_phase_change(fcpl, 0, 0);
-#endif /* QAK */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Creating file\n");
-
- /* Create the file */
- if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
- return -1;
-
- /* Close file creation property list */
- if(H5Pclose(fcpl) < 0)
- return -1;
-
- /* Close file access property list */
- if(H5Pclose(fapl) < 0)
- return -1;
-
- /* Create attribute with (shared) random number seed - for sparse test */
- if((sid = H5Screate(H5S_SCALAR)) < 0)
- return -1;
- if((aid = H5Acreate2(fid, "seed", H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
- return -1;
- if(H5Awrite(aid, H5T_NATIVE_UINT, &random_seed) < 0)
- return -1;
- if(H5Sclose(sid) < 0)
- return -1;
-
- /* Create datatype for creating datasets */
- if((tid = create_symbol_datatype()) < 0)
- return -1;
-
- /* Create dataspace for creating datasets */
- if((sid = H5Screate_simple(2, dims, max_dims)) < 0)
- return -1;
-
- /* Create dataset creation property list */
- if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
- return -1;
- if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0)
- return -1;
- if(comp_level >= 0) {
- if(H5Pset_deflate(dcpl, (unsigned)comp_level) < 0)
- return -1;
- } /* end if */
-#ifdef FILLVAL_WORKS
- /* Currently fill values do not work because they can bump the dataspace
- * message to the second object header chunk. We should enable the fillval
- * here when this is fixed. -NAF 8/11/11 */
- memset(&fillval, 0, sizeof(fillval));
- fillval.rec_id = (uint64_t)ULLONG_MAX;
- if(H5Pset_fill_value(dcpl, tid, &fillval) < 0)
- return -1;
-#endif /* FILLVAL_WORKS */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Creating datasets\n");
-
- /* Create the datasets */
- for(u = 0; u < NLEVELS; u++)
- for(v = 0; v < symbol_count[u]; v++) {
- hid_t dsid; /* Dataset ID */
- char name_buf[64];
-
- generate_name(name_buf, u, v);
- if((dsid = H5Dcreate2(fid, name_buf, tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
- return -1;
-
- if(H5Dclose(dsid) < 0)
- return -1;
- } /* end for */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Closing objects\n");
-
- /* Close everythign */
- if(H5Pclose(dcpl) < 0)
- return -1;
- if(H5Sclose(sid) < 0)
- return -1;
- if(H5Tclose(tid) < 0)
- return -1;
- if(H5Fclose(fid) < 0)
- return -1;
-
- return 0;
-} /* end gen_skeleton() */
-
-static void
-usage(void)
-{
- printf("Usage error!\n");
- printf("Usage: swmr_generator [-q] [-c <deflate compression level>] [-i <index type>] [-r <random # seed>]\n");
- printf("NOTE: The random seed option is only used by the sparse test\n");
- printf("<deflate compression level> should be -1 (for no compression) or 0-9\n");
- printf("<index type> should be b1, b2, fa, or ea (fa not yet implemented)\n");
- printf("Defaults to verbose (no '-q' given), no compression ('-c -1') and v1 b-tree\n");
- printf(" (-i b1)");
- exit(1);
-} /* end usage() */
-
-int main(int argc, const char *argv[])
-{
- int comp_level = -1; /* Compression level (-1 is no compression) */
- unsigned verbose = 1; /* Whether to emit some informational messages */
- const char *index_type = "b1"; /* Chunk index type */
- unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */
- unsigned random_seed = 0; /* Random # seed */
- unsigned u; /* Local index variables */
- int temp;
-
- /* Parse command line options */
- if(argc > 1) {
- u = 1;
- while(u < (unsigned)argc) {
- if(argv[u][0] == '-') {
- switch(argv[u][1]) {
- /* Compress dataset chunks */
- case 'c':
- comp_level = atoi(argv[u + 1]);
- if(comp_level < -1 || comp_level > 9)
- usage();
- u += 2;
- break;
-
- /* Chunk index type */
- case 'i':
- index_type = argv[u + 1];
- if(strcmp(index_type, "b1")
- && strcmp(index_type, "ea")
- && strcmp(index_type, "b2"))
- usage();
- u += 2;
- break;
-
- /* Random # seed */
- case 'r':
- use_seed = 1;
- temp = atoi(argv[u + 1]);
- if(temp < 0)
- usage();
- else
- random_seed = (unsigned)temp;
- u += 2;
- break;
-
- /* Be quiet */
- case 'q':
- verbose = 0;
- u++;
- break;
-
- default:
- usage();
- break;
- } /* end switch */
- } /* end if */
- } /* end while */
- } /* end if */
-
- /* Emit informational message */
- if(verbose) {
- fprintf(stderr, "Parameters:\n");
- fprintf(stderr, "\tcompression level = %d\n", comp_level);
- fprintf(stderr, "\tindex_type = %s\n", index_type);
- } /* end if */
-
- /* Set the random seed */
- if(0 == use_seed) {
- struct timeval t;
- gettimeofday(&t, NULL);
- random_seed = (unsigned)((t.tv_sec * 1000) + t.tv_usec);
- } /* end if */
- srandom(random_seed);
- /* ALWAYS emit the random seed for possible debugging */
- fprintf(stderr, "Using generator random seed (used in sparse test only): %u\n", random_seed);
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Generating skeleton file: %s\n", FILENAME);
-
- /* Generate file skeleton */
- if(gen_skeleton(FILENAME, verbose, comp_level, index_type, random_seed) < 0) {
- fprintf(stderr, "Error generating skeleton file!\n");
- exit(1);
- } /* end if */
-
- return 0;
-}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: swmr_generator.c + * + * Purpose: Functions for building and setting up the SWMR test file + * and datasets. + * + *------------------------------------------------------------------------- + */ + + /***********/ +/* Headers */ +/***********/ + +#include <assert.h> +#include <sys/time.h> + +#include "swmr_common.h" + +/****************/ +/* Local Macros */ +/****************/ + +#define CHUNK_SIZE 50 /* Chunk size for created datasets */ + +/********************/ +/* Local Prototypes */ +/********************/ + +static int gen_skeleton(const char *filename, unsigned verbose, int comp_level, + const char *index_type, unsigned random_seed); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: gen_skeleton + * + * Purpose: Creates the HDF5 file and datasets which will be used in + * the SWMR testing. + * + * Parameters: const char *filename + * The SWMR test file's name. + * + * unsigned verbose + * Whether verbose console output is desired. + * + * int comp_level + * The zlib compression level to use. -1 = no compression. + * + * const char *index_type + * The chunk index type (b1 | b2 | ea | fa) + * + * unsigned random_seed + * The random seed to store in the file. The sparse tests use + * this value. + * + * Return: Success: 0 + * + * Failure: Can't fail + * + *------------------------------------------------------------------------- + */ +static int +gen_skeleton(const char *filename, unsigned verbose, int comp_level, + const char *index_type, unsigned random_seed) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File access property list */ + hid_t dcpl; /* Dataset creation property list */ + hid_t tid; /* Datatype for dataset elements */ + hid_t sid; /* Dataspace ID */ + hid_t aid; /* Attribute ID */ + hsize_t dims[2] = {1, 0}; /* Dataset starting dimensions */ + hsize_t max_dims[2] = {1, H5S_UNLIMITED}; /* Dataset maximum dimensions */ + hsize_t chunk_dims[2] = {1, CHUNK_SIZE}; /* Chunk dimensions */ +#ifdef FILLVAL_WORKS + symbol_t fillval; /* Dataset fill value */ +#endif /* FILLVAL_WORKS */ + unsigned u, v; /* Local index variable */ + + assert(filename); + assert(index_type); + + /* Create file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return -1; + + /* Select the correct index type */ + if(strcmp(index_type, "b1")) + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + return -1; + if(!strcmp(index_type, "b2")) + max_dims[0] = H5S_UNLIMITED; + +#ifdef QAK + /* Increase the initial size of the metadata cache */ + { + H5AC_cache_config_t mdc_config; + + mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + H5Pget_mdc_config(fapl, &mdc_config); + fprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); + fprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); + mdc_config.set_initial_size = 1; + mdc_config.initial_size = 16 * 1024 * 1024; + /* mdc_config.epoch_length = 5000; */ + H5Pset_mdc_config(fapl, &mdc_config); + } +#endif /* QAK */ + +#ifdef QAK + H5Pset_small_data_block_size(fapl, (hsize_t)(50 * CHUNK_SIZE * DTYPE_SIZE)); +#endif /* QAK */ + +#ifdef QAK + H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); +#endif /* QAK */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + +#ifdef QAK + H5Pset_link_phase_change(fcpl, 0, 0); +#endif /* QAK */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Creating file\n"); + + /* Create the file */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + + /* Close file creation property list */ + if(H5Pclose(fcpl) < 0) + return -1; + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + return -1; + + /* Create attribute with (shared) random number seed - for sparse test */ + if((sid = H5Screate(H5S_SCALAR)) < 0) + return -1; + if((aid = H5Acreate2(fid, "seed", H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + return -1; + if(H5Awrite(aid, H5T_NATIVE_UINT, &random_seed) < 0) + return -1; + if(H5Sclose(sid) < 0) + return -1; + + /* Create datatype for creating datasets */ + if((tid = create_symbol_datatype()) < 0) + return -1; + + /* Create dataspace for creating datasets */ + if((sid = H5Screate_simple(2, dims, max_dims)) < 0) + return -1; + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + return -1; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + return -1; + if(comp_level >= 0) { + if(H5Pset_deflate(dcpl, (unsigned)comp_level) < 0) + return -1; + } /* end if */ +#ifdef FILLVAL_WORKS + /* Currently fill values do not work because they can bump the dataspace + * message to the second object header chunk. We should enable the fillval + * here when this is fixed. -NAF 8/11/11 */ + memset(&fillval, 0, sizeof(fillval)); + fillval.rec_id = (uint64_t)ULLONG_MAX; + if(H5Pset_fill_value(dcpl, tid, &fillval) < 0) + return -1; +#endif /* FILLVAL_WORKS */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Creating datasets\n"); + + /* Create the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + hid_t dsid; /* Dataset ID */ + char name_buf[64]; + + generate_name(name_buf, u, v); + if((dsid = H5Dcreate2(fid, name_buf, tid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + return -1; + + if(H5Dclose(dsid) < 0) + return -1; + } /* end for */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Closing objects\n"); + + /* Close everythign */ + if(H5Pclose(dcpl) < 0) + return -1; + if(H5Sclose(sid) < 0) + return -1; + if(H5Tclose(tid) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + + return 0; +} /* end gen_skeleton() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: swmr_generator [-q] [-c <deflate compression level>] [-i <index type>]\n"); + printf(" [-r <random seed>]\n"); + printf("\n"); + printf("NOTE: The random seed option is only used by the sparse test. Other\n"); + printf(" tests specify the random seed as a reader/writer option.\n"); + printf("\n"); + printf("<deflate compression level> should be -1 (for no compression) or 0-9\n"); + printf("\n"); + printf("<index type> should be b1, b2, fa, or ea (fa not yet implemented)\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), no compression ('-c -1'), v1 b-tree\n"); + printf("indexing (-i b1), and will generate a random seed (no -r given).\n"); + printf("\n"); + exit(1); +} /* end usage() */ + +int main(int argc, const char *argv[]) +{ + int comp_level = -1; /* Compression level (-1 is no compression) */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + const char *index_type = "b1"; /* Chunk index type */ + unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variables */ + int temp; + + /* Parse command line options */ + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* Compress dataset chunks */ + case 'c': + comp_level = atoi(argv[u + 1]); + if(comp_level < -1 || comp_level > 9) + usage(); + u += 2; + break; + + /* Chunk index type */ + case 'i': + index_type = argv[u + 1]; + if(strcmp(index_type, "b1") + && strcmp(index_type, "ea") + && strcmp(index_type, "b2")) + usage(); + u += 2; + break; + + /* Random # seed */ + case 'r': + use_seed = 1; + temp = atoi(argv[u + 1]); + if(temp < 0) + usage(); + else + random_seed = (unsigned)temp; + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + } /* end while */ + } /* end if */ + + /* Emit informational message */ + if(verbose) { + fprintf(stderr, "Parameters:\n"); + fprintf(stderr, "\tcompression level = %d\n", comp_level); + fprintf(stderr, "\tindex_type = %s\n", index_type); + } /* end if */ + + /* Set the random seed */ + if(0 == use_seed) { + struct timeval t; + gettimeofday(&t, NULL); + random_seed = (unsigned)((t.tv_sec * 1000) + t.tv_usec); + } /* end if */ + srandom(random_seed); + /* ALWAYS emit the random seed for possible debugging */ + fprintf(stderr, "Using generator random seed (used in sparse test only): %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Generating skeleton file: %s\n", FILENAME); + + /* Generate file skeleton */ + if(gen_skeleton(FILENAME, verbose, comp_level, index_type, random_seed) < 0) { + fprintf(stderr, "Error generating skeleton file!\n"); + exit(1); + } /* end if */ + + return 0; +} diff --git a/test/swmr_reader.c b/test/swmr_reader.c index 02eaf9f..14e91ca 100644 --- a/test/swmr_reader.c +++ b/test/swmr_reader.c @@ -1,381 +1,500 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * Copyright by The HDF Group. *
- * Copyright by the Board of Trustees of the University of Illinois. *
- * All rights reserved. *
- * *
- * This file is part of HDF5. The full HDF5 copyright notice, including *
- * terms governing use, modification, and redistribution, is contained in *
- * the files COPYING and Copyright.html. COPYING can be found at the root *
- * of the source code distribution tree; Copyright.html can be found at the *
- * root level of an installed copy of the electronic HDF5 document set and *
- * is linked from the top-level documents page. It can also be found at *
- * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
- * access to either file, you may request a copy from help@hdfgroup.org. *
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-#include <unistd.h>
-#include <sys/time.h>
-
-#include "swmr_common.h"
-
-static hid_t symbol_tid = -1;
-
-static int
-check_dataset(hid_t fid, unsigned verbose, const char *sym_name, symbol_t *record,
- hid_t rec_sid)
-{
- hid_t dsid; /* Dataset ID */
- hid_t file_sid; /* Dataset's space ID */
- hssize_t snpoints; /* Number of elements in dataset */
- hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
-
- /* Open dataset for symbol */
- if((dsid = H5Dopen2(fid, sym_name, H5P_DEFAULT)) < 0)
- return -1;
-
- /* Get the dataset's dataspace */
- if((file_sid = H5Dget_space(dsid)) < 0)
- return -1;
-
- /* Get the number of elements (= records, for 1-D datasets) */
- if((snpoints = H5Sget_simple_extent_npoints(file_sid)) < 0)
- return -1;
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Symbol = '%s', # of records = %lld\n", sym_name, (long long)snpoints);
-
- /* Check if there are records for symbol */
- if(snpoints > 0) {
- /* Choose the last record in the dataset */
- start[1] = (hsize_t)(snpoints - 1);
- if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
- return -1;
-
- /* Read record from dataset */
- record->rec_id = (uint64_t)ULLONG_MAX;
- if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0)
- return -1;
-
- /* Verify record value */
- if(record->rec_id != start[1]) {
- fprintf(stderr, "*** ERROR ***\n");
- fprintf(stderr, "Incorrect record value!\n");
- fprintf(stderr, "Symbol = '%s', # of records = %lld, record->rec_id = %llu\n", sym_name, (long long)snpoints, (unsigned long long)record->rec_id);
- return -1;
- } /* end if */
- } /* end if */
-
- /* Close the dataset's dataspace */
- if(H5Sclose(file_sid) < 0)
- return -1;
-
- /* Close dataset for symbol */
- if(H5Dclose(dsid) < 0)
- return -1;
-
- return 0;
-} /* end check_dataset() */
-
-static int
-read_records(const char *filename, unsigned verbose, unsigned long nseconds,
- unsigned poll_time, unsigned ncommon, unsigned nrandom)
-{
- time_t start_time; /* Starting time */
- time_t curr_time; /* Current time */
- symbol_info_t **sym_com = NULL, **sym_rand = NULL; /* Pointers to arrays of common & random dataset IDs */
- hid_t mem_sid; /* Memory dataspace ID */
- symbol_t record; /* The record to add to the dataset */
- unsigned v; /* Local index variable */
-
- /* Reset the record */
- /* (record's 'info' field might need to change for each record written, also) */
- memset(&record, 0, sizeof(record));
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Choosing datasets\n");
-
- /* Allocate space for 'common' datasets, if any */
- if(ncommon > 0) {
- /* Allocate array to hold pointers to symbols for common datasets */
- if(NULL == (sym_com = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * ncommon)))
- return -1;
-
- /* Open the common datasets */
- for(v = 0; v < ncommon; v++) {
- unsigned offset; /* Offset of symbol to use */
-
- /* Determine the offset of the symbol, within level 0 symbols */
- /* (level 0 symbols are the most common symbols) */
- offset = (unsigned)(random() % symbol_count[0]);
- sym_com[v] = &symbol_info[0][offset];
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Common symbol #%u = '%s'\n", v, symbol_info[0][offset].name);
- } /* end for */
- } /* end if */
-
- /* Allocate space for 'random' datasets, if any */
- if(nrandom > 0) {
- /* Allocate array to hold pointers to symbols for random datasets */
- if(NULL == (sym_rand = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * nrandom)))
- return -1;
-
- /* Determine the random datasets */
- for(v = 0; v < nrandom; v++) {
- symbol_info_t *sym; /* Symbol to use */
-
- /* Determine the symbol, within all symbols */
- if(NULL == (sym = choose_dataset()))
- return -1;
- sym_rand[v] = sym;
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Random symbol #%u = '%s'\n", v, sym->name);
- } /* end for */
- } /* end if */
-
- /* Create a dataspace for the record to read */
- if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
- return -1;
-
- /* Emit informational message */
- if(verbose)
- printf("Reading records\n");
-
- /* Get the starting time */
- start_time = time(NULL);
- curr_time = start_time;
-
- /* Loop over reading records until [at least] the correct # of seconds have passed */
- while(curr_time < (time_t)(start_time + (time_t)nseconds)) {
- hid_t fid; /* File ID */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Opening file: %s\n", filename);
-
- /* Open the file */
- if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT)) < 0)
- return -1;
-
- /* Check 'common' datasets, if any */
- if(ncommon > 0) {
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Checking common symbols\n");
-
- /* Iterate over common datasets */
- for(v = 0; v < ncommon; v++) {
- /* Check common dataset */
- if(check_dataset(fid, verbose, sym_com[v]->name, &record, mem_sid) < 0)
- return -1;
- } /* end for */
- } /* end if */
-
- /* Check 'random' datasets, if any */
- if(nrandom > 0) {
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Checking random symbols\n");
-
- /* Iterate over random datasets */
- for(v = 0; v < nrandom; v++) {
- /* Check random dataset */
- if(check_dataset(fid, verbose, sym_rand[v]->name, &record, mem_sid) < 0)
- return -1;
- } /* end for */
- } /* end if */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Closing file\n");
-
- /* Close the file */
- if(H5Fclose(fid) < 0)
- return -1;
-
- /* Sleep for the appropriate # of seconds */
- sleep(poll_time);
-
- /* Retrieve the current time */
- curr_time = time(NULL);
- } /* end while */
-
- /* Close the memory dataspace */
- if(H5Sclose(mem_sid) < 0)
- return -1;
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Closing datasets\n");
-
- /* Close 'random' datasets, if any */
- if(nrandom > 0) {
- /* Release array holding dataset ID's for random datasets */
- free(sym_rand);
- } /* end if */
-
- /* Close 'common' datasets, if any */
- if(ncommon > 0) {
- /* Release array holding dataset ID's for common datasets */
- free(sym_com);
- } /* end if */
-
- return 0;
-} /* end read_records() */
-
-static void
-usage(void)
-{
- printf("Usage error!\n");
- printf("Usage: swmr_reader [-q] [-s <# of seconds to sleep between polling>] [-h <# of common symbols to poll>] [-l <# of random symbols to poll>] [-r <random # seed>] <# of seconds to test>\n");
- printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'), 5 common symbols to poll ('-h 5') and 10 random symbols to poll ('-l 10')\n");
- exit(1);
-}
-
-int main(int argc, const char *argv[])
-{
- long nseconds = 0; /* # of seconds to test */
- int poll_time = 1; /* # of seconds between polling */
- int ncommon = 5; /* # of common symbols to poll */
- int nrandom = 10; /* # of random symbols to poll */
- unsigned verbose = 1; /* Whether to emit some informational messages */
- unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */
- unsigned random_seed = 0; /* Random # seed */
- unsigned u; /* Local index variables */
- int temp;
-
- /* Parse command line options */
- if(argc < 2)
- usage();
- if(argc > 1) {
- u = 1;
- while(u < (unsigned)argc) {
- if(argv[u][0] == '-') {
- switch(argv[u][1]) {
- /* # of common symbols to poll */
- case 'h':
- ncommon = atoi(argv[u + 1]);
- if(ncommon < 0)
- usage();
- u += 2;
- break;
-
- /* # of random symbols to poll */
- case 'l':
- nrandom = atoi(argv[u + 1]);
- if(nrandom < 0)
- usage();
- u += 2;
- break;
-
- /* Be quiet */
- case 'q':
- verbose = 0;
- u++;
- break;
-
- /* Random # seed */
- case 'r':
- use_seed = 1;
- temp = atoi(argv[u + 1]);
- if(temp < 0)
- usage();
- else
- random_seed = (unsigned)temp;
- u += 2;
- break;
-
- /* # of seconds between polling */
- case 's':
- poll_time = atoi(argv[u + 1]);
- if(poll_time < 0)
- usage();
- u += 2;
- break;
-
- default:
- usage();
- break;
- } /* end switch */
- } /* end if */
- else {
- /* Get the number of records to append */
- nseconds = atol(argv[u]);
- if(nseconds <= 0)
- usage();
-
- u++;
- } /* end else */
- } /* end while */
- } /* end if */
- if(nseconds <= 0)
- usage();
- if(poll_time >= nseconds)
- usage();
-
- /* Emit informational message */
- if(verbose) {
- fprintf(stderr, "Parameters:\n");
- fprintf(stderr, "\t# of seconds between polling = %d\n", poll_time);
- fprintf(stderr, "\t# of common symbols to poll = %d\n", ncommon);
- fprintf(stderr, "\t# of random symbols to poll = %d\n", nrandom);
- fprintf(stderr, "\t# of seconds to test = %ld\n", nseconds);
- } /* end if */
-
- /* Set the random seed */
- if(0 == use_seed) {
- struct timeval t;
- gettimeofday(&t, NULL);
- random_seed = (unsigned)((t.tv_sec * 1000) + t.tv_usec);
- } /* end if */
- srandom(random_seed);
- /* ALWAYS emit the random seed for possible debugging */
- fprintf(stderr, "Using reader random seed: %u\n", random_seed);
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Generating symbol names\n");
-
- /* Generate dataset names */
- if(generate_symbols() < 0) {
- fprintf(stderr, "Error generating symbol names!\n");
- exit(1);
- } /* end if */
-
- /* Create datatype for creating datasets */
- if((symbol_tid = create_symbol_datatype()) < 0)
- return -1;
-
- /* Reading records from datasets */
- if(read_records(FILENAME, verbose, (unsigned long)nseconds, (unsigned)poll_time, (unsigned)ncommon, (unsigned)nrandom) < 0) {
- fprintf(stderr, "Error reading records from datasets!\n");
- exit(1);
- } /* end if */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Releasing symbols\n");
-
- /* Clean up the symbols */
- if(shutdown_symbols() < 0) {
- fprintf(stderr, "Error releasing symbols!\n");
- exit(1);
- } /* end if */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Closing objects\n");
-
- /* Close objects created */
- if(H5Tclose(symbol_tid) < 0) {
- fprintf(stderr, "Error closing symbol datatype!\n");
- exit(1);
- } /* end if */
-
- return 0;
-}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: swmr_reader.c + * + * Purpose: Reads data from a randomly selected subset of the datasets + * in the SWMR test file. + * + * This program is intended to run concurrently with the + * swmr_writer program. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <assert.h> +#include <unistd.h> +#include <sys/time.h> + +#include "swmr_common.h" + +/********************/ +/* Local Prototypes */ +/********************/ + +static int check_dataset(hid_t fid, unsigned verbose, const char *sym_name, + symbol_t *record, hid_t rec_sid); +static int read_records(const char *filename, unsigned verbose, unsigned long nseconds, + unsigned poll_time, unsigned ncommon, unsigned nrandom); + +/*******************/ +/* Local Variables */ +/*******************/ + +static hid_t symbol_tid = -1; /* The type ID for the SWMR datasets */ + + +/*------------------------------------------------------------------------- + * Function: check_dataset + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * Parameters: hid_t fid + * The SWMR test file's ID. + * + * unsigned verbose + * Whether verbose console output is desired. + * + * const char *sym_name + * The name of the dataset from which to read. + * + * symbol_t *record + * Memory for the record. Must be pre-allocated. + * + * hid_t rec_sid + * The memory dataspace for access. It's always the same so + * there is no need to re-create it every time this function + * is called. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +check_dataset(hid_t fid, unsigned verbose, const char *sym_name, symbol_t *record, + hid_t rec_sid) +{ + hid_t dsid; /* Dataset ID */ + hid_t file_sid; /* Dataset's space ID */ + hssize_t snpoints; /* Number of elements in dataset */ + hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */ + + assert(fid >= 0); + assert(sym_name); + assert(record); + assert(rec_sid >= 0); + + /* Open dataset for symbol */ + if((dsid = H5Dopen2(fid, sym_name, H5P_DEFAULT)) < 0) + return -1; + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(dsid)) < 0) + return -1; + + /* Get the number of elements (= records, for 1-D datasets) */ + if((snpoints = H5Sget_simple_extent_npoints(file_sid)) < 0) + return -1; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Symbol = '%s', # of records = %lld\n", sym_name, (long long)snpoints); + + /* Check if there are records for symbol */ + if(snpoints > 0) { + /* Choose the last record in the dataset */ + start[1] = (hsize_t)(snpoints - 1); + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + return -1; + + /* Read record from dataset */ + record->rec_id = (uint64_t)ULLONG_MAX; + if(H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0) + return -1; + + /* Verify record value */ + if(record->rec_id != start[1]) { + fprintf(stderr, "*** ERROR ***\n"); + fprintf(stderr, "Incorrect record value!\n"); + fprintf(stderr, "Symbol = '%s', # of records = %lld, record->rec_id = %llu\n", sym_name, (long long)snpoints, (unsigned long long)record->rec_id); + return -1; + } /* end if */ + } /* end if */ + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + return -1; + + /* Close dataset for symbol */ + if(H5Dclose(dsid) < 0) + return -1; + + return 0; +} /* end check_dataset() */ + + +/*------------------------------------------------------------------------- + * Function: read_records + * + * Purpose: For a given dataset, checks to make sure that the stated + * and actual sizes are the same. If they are not, then + * we have an inconsistent dataset due to a SWMR error. + * + * The "common" datasets are a random selection from among + * the level 0 datasets. The "random" datasets are a random + * selection from among all the file's datasets. This scheme + * ensures that the level 0 datasets are interrogated vigorously. + * + * Parameters: const char *filename + * The SWMR test file's name. + * + * unsigned verbose + * Whether verbose console output is desired. + * + * unsigned long nseconds + * The amount of time to read records (ns). + * + * unsigned poll_time + * The amount of time to sleep (s). + * + * unsigned ncommon + * The number of common/non-random datasets that will be opened. + * + * unsigned nrandom + * The number of random datasets that will be opened. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +read_records(const char *filename, unsigned verbose, unsigned long nseconds, + unsigned poll_time, unsigned ncommon, unsigned nrandom) +{ + time_t start_time; /* Starting time */ + time_t curr_time; /* Current time */ + symbol_info_t **sym_com = NULL; /* Pointers to array of common dataset IDs */ + symbol_info_t **sym_rand = NULL; /* Pointers to array of random dataset IDs */ + hid_t mem_sid; /* Memory dataspace ID */ + hid_t fid; /* SWMR test file ID */ + symbol_t record; /* The record to read from the dataset */ + unsigned v; /* Local index variable */ + + assert(filename); + assert(nseconds != 0); + assert(poll_time != 0); + + /* Reset the record */ + /* (record's 'info' field might need to change for each record read, also) */ + memset(&record, 0, sizeof(record)); + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Choosing datasets\n"); + + /* Allocate space for 'common' datasets, if any */ + if(ncommon > 0) { + /* Allocate array to hold pointers to symbols for common datasets */ + if(NULL == (sym_com = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * ncommon))) + return -1; + + /* Open the common datasets */ + for(v = 0; v < ncommon; v++) { + unsigned offset; /* Offset of symbol to use */ + + /* Determine the offset of the symbol, within level 0 symbols */ + /* (level 0 symbols are the most common symbols) */ + offset = (unsigned)(random() % symbol_count[0]); + sym_com[v] = &symbol_info[0][offset]; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Common symbol #%u = '%s'\n", v, symbol_info[0][offset].name); + } /* end for */ + } /* end if */ + + /* Allocate space for 'random' datasets, if any */ + if(nrandom > 0) { + /* Allocate array to hold pointers to symbols for random datasets */ + if(NULL == (sym_rand = (symbol_info_t **)malloc(sizeof(symbol_info_t *) * nrandom))) + return -1; + + /* Determine the random datasets */ + for(v = 0; v < nrandom; v++) { + symbol_info_t *sym; /* Symbol to use */ + + /* Determine the symbol, within all symbols */ + if(NULL == (sym = choose_dataset())) + return -1; + sym_rand[v] = sym; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Random symbol #%u = '%s'\n", v, sym->name); + } /* end for */ + } /* end if */ + + /* Create a dataspace for the record to read */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + return -1; + + /* Emit informational message */ + if(verbose) + printf("Reading records\n"); + + /* Get the starting time */ + start_time = time(NULL); + curr_time = start_time; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Opening file: %s\n", filename); + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT)) < 0) + return -1; + + /* Loop over reading records until [at least] the correct # of seconds have passed */ + while(curr_time < (time_t)(start_time + (time_t)nseconds)) { + + /* Check 'common' datasets, if any */ + if(ncommon > 0) { + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Checking common symbols\n"); + + /* Iterate over common datasets */ + for(v = 0; v < ncommon; v++) { + /* Check common dataset */ + if(check_dataset(fid, verbose, sym_com[v]->name, &record, mem_sid) < 0) + return -1; + memset(&record, 0, sizeof(record)); + } /* end for */ + } /* end if */ + + /* Check 'random' datasets, if any */ + if(nrandom > 0) { + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Checking random symbols\n"); + + /* Iterate over random datasets */ + for(v = 0; v < nrandom; v++) { + /* Check random dataset */ + if(check_dataset(fid, verbose, sym_rand[v]->name, &record, mem_sid) < 0) + return -1; + memset(&record, 0, sizeof(record)); + } /* end for */ + } /* end if */ + + /* Sleep for the appropriate # of seconds */ + sleep(poll_time); + + /* Retrieve the current time */ + curr_time = time(NULL); + } /* end while */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Closing file\n"); + + /* Close the file */ + if(H5Fclose(fid) < 0) + return -1; + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + return -1; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Closing datasets\n"); + + /* Close 'random' datasets, if any */ + if(nrandom > 0) { + /* Release array holding dataset ID's for random datasets */ + free(sym_rand); + } /* end if */ + + /* Close 'common' datasets, if any */ + if(ncommon > 0) { + /* Release array holding dataset ID's for common datasets */ + free(sym_com); + } /* end if */ + + return 0; +} /* end read_records() */ + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: swmr_reader [-q] [-s <# of seconds to sleep between polling>]\n"); + printf(" [-h <# of common symbols to poll>] [-l <# of random symbols to poll>]\n"); + printf(" [-r <random seed>] <# of seconds to test>\n"); + printf("\n"); + printf("<# of seconds to test> must be specified.\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'),\n"); + printf("5 common symbols to poll ('-h 5'), 10 random symbols to poll ('-l 10'),\n"); + printf("and will generate a random seed (no -r given).\n"); + printf("\n"); + exit(1); +} + +int main(int argc, const char *argv[]) +{ + long nseconds = 0; /* # of seconds to test */ + int poll_time = 1; /* # of seconds between polling */ + int ncommon = 5; /* # of common symbols to poll */ + int nrandom = 10; /* # of random symbols to poll */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variables */ + int temp; + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of common symbols to poll */ + case 'h': + ncommon = atoi(argv[u + 1]); + if(ncommon < 0) + usage(); + u += 2; + break; + + /* # of random symbols to poll */ + case 'l': + nrandom = atoi(argv[u + 1]); + if(nrandom < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + /* Random # seed */ + case 'r': + use_seed = 1; + temp = atoi(argv[u + 1]); + if(temp < 0) + usage(); + else + random_seed = (unsigned)temp; + u += 2; + break; + + /* # of seconds between polling */ + case 's': + poll_time = atoi(argv[u + 1]); + if(poll_time < 0) + usage(); + u += 2; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nseconds = atol(argv[u]); + if(nseconds <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nseconds <= 0) + usage(); + if(poll_time >= nseconds) + usage(); + + /* Emit informational message */ + if(verbose) { + fprintf(stderr, "Parameters:\n"); + fprintf(stderr, "\t# of seconds between polling = %d\n", poll_time); + fprintf(stderr, "\t# of common symbols to poll = %d\n", ncommon); + fprintf(stderr, "\t# of random symbols to poll = %d\n", nrandom); + fprintf(stderr, "\t# of seconds to test = %ld\n", nseconds); + } /* end if */ + + /* Set the random seed */ + if(0 == use_seed) { + struct timeval t; + gettimeofday(&t, NULL); + random_seed = (unsigned)((t.tv_sec * 1000) + t.tv_usec); + } /* end if */ + srandom(random_seed); + /* ALWAYS emit the random seed for possible debugging */ + fprintf(stderr, "Using reader random seed: %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) { + fprintf(stderr, "Error generating symbol names!\n"); + exit(1); + } /* end if */ + + /* Create datatype for creating datasets */ + if((symbol_tid = create_symbol_datatype()) < 0) + return -1; + + /* Reading records from datasets */ + if(read_records(FILENAME, verbose, (unsigned long)nseconds, (unsigned)poll_time, (unsigned)ncommon, (unsigned)nrandom) < 0) { + fprintf(stderr, "Error reading records from datasets!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + fprintf(stderr, "Error releasing symbols!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Closing objects\n"); + + /* Close objects created */ + if(H5Tclose(symbol_tid) < 0) { + fprintf(stderr, "Error closing symbol datatype!\n"); + exit(1); + } /* end if */ + + return 0; +} diff --git a/test/swmr_remove_reader.c b/test/swmr_remove_reader.c index 26c52d8..e8a1194 100644 --- a/test/swmr_remove_reader.c +++ b/test/swmr_remove_reader.c @@ -13,13 +13,76 @@ * access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*-------------------------------------------------------------------------
+ *
+ * Created: swmr_remove_reader.c
+ *
+ * Purpose: Reads data from a randomly selected subset of the datasets
+ * in the SWMR test file. Unlike the regular reader, these
+ * datasets will be shrinking.
+ *
+ * This program is intended to run concurrently with the
+ * swmr_remove_writer program.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include <assert.h>
#include <unistd.h>
#include <sys/time.h>
#include "swmr_common.h"
+/*******************/
+/* Local Variables */
+/*******************/
+
static hid_t symbol_tid = -1;
+/********************/
+/* Local Prototypes */
+/********************/
+
+static int check_dataset(hid_t fid, unsigned verbose, const char *sym_name,
+ symbol_t *record, hid_t rec_sid);
+static int read_records(const char *filename, unsigned verbose, unsigned long nseconds,
+ unsigned poll_time, unsigned ncommon, unsigned nrandom);
+static void usage(void);
+
+
+/*-------------------------------------------------------------------------
+ * Function: check_dataset
+ *
+ * Purpose: For a given dataset, checks to make sure that the stated
+ * and actual sizes are the same. If they are not, then
+ * we have an inconsistent dataset due to a SWMR error.
+ *
+ * Parameters: hid_t fid
+ * The SWMR test file's ID.
+ *
+ * unsigned verbose
+ * Whether verbose console output is desired.
+ *
+ * const char *sym_name
+ * The name of the dataset from which to read.
+ *
+ * symbol_t *record
+ * Memory for the record. Must be pre-allocated.
+ *
+ * hid_t rec_sid
+ * The memory dataspace for access. It's always the same so
+ * there is no need to re-create it every time this function
+ * is called.
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static int
check_dataset(hid_t fid, unsigned verbose, const char *sym_name, symbol_t *record,
hid_t rec_sid)
@@ -29,6 +92,11 @@ check_dataset(hid_t fid, unsigned verbose, const char *sym_name, symbol_t *recor hssize_t snpoints; /* Number of elements in dataset */
hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
+ assert(fid >= 0);
+ assert(sym_name);
+ assert(record);
+ assert(rec_sid >= 0);
+
/* Open dataset for symbol */
if((dsid = H5Dopen2(fid, sym_name, H5P_DEFAULT)) < 0)
return -1;
@@ -90,17 +158,59 @@ check_dataset(hid_t fid, unsigned verbose, const char *sym_name, symbol_t *recor return 0;
} /* end check_dataset() */
+
+/*-------------------------------------------------------------------------
+ * Function: read_records
+ *
+ * Purpose: For a given dataset, checks to make sure that the stated
+ * and actual sizes are the same. If they are not, then
+ * we have an inconsistent dataset due to a SWMR error.
+ *
+ * The "common" datasets are a random selection from among
+ * the level 0 datasets. The "random" datasets are a random
+ * selection from among all the file's datasets. This scheme
+ * ensures that the level 0 datasets are interrogated vigorously.
+ *
+ * Parameters: const char *filename
+ * The SWMR test file's name.
+ *
+ * unsigned verbose
+ * Whether verbose console output is desired.
+ *
+ * unsigned long nseconds
+ * The amount of time to read records (ns).
+ *
+ * unsigned poll_time
+ * The amount of time to sleep (s).
+ *
+ * unsigned ncommon
+ * The number of common/non-random datasets that will be opened.
+ *
+ * unsigned nrandom
+ * The number of random datasets that will be opened.
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static int
read_records(const char *filename, unsigned verbose, unsigned long nseconds,
unsigned poll_time, unsigned ncommon, unsigned nrandom)
{
time_t start_time; /* Starting time */
time_t curr_time; /* Current time */
- symbol_info_t **sym_com = NULL, **sym_rand = NULL; /* Pointers to arrays of common & random dataset IDs */
+ symbol_info_t **sym_com = NULL; /* Pointers to array of common dataset IDs */
+ symbol_info_t **sym_rand = NULL; /* Pointers to array of random dataset IDs */
hid_t mem_sid; /* Memory dataspace ID */
+ hid_t fid; /* SWMR test file ID */
symbol_t record; /* The record to add to the dataset */
unsigned v; /* Local index variable */
+ assert(filename);
+ assert(nseconds != 0);
+ assert(poll_time != 0);
+
/* Reset the record */
/* (record's 'info' field might need to change for each record written, also) */
memset(&record, 0, sizeof(record));
@@ -163,17 +273,16 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, start_time = time(NULL);
curr_time = start_time;
- /* Loop over reading records until [at least] the correct # of seconds have passed */
- while(curr_time < (time_t)(start_time + (time_t)nseconds)) {
- hid_t fid; /* File ID */
+ /* Emit informational message */
+ if(verbose)
+ fprintf(stderr, "Opening file: %s\n", filename);
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Opening file: %s\n", filename);
+ /* Open the file */
+ if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT)) < 0)
+ return -1;
- /* Open the file */
- if((fid = H5Fopen(filename, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, H5P_DEFAULT)) < 0)
- return -1;
+ /* Loop over reading records until [at least] the correct # of seconds have passed */
+ while(curr_time < (time_t)(start_time + (time_t)nseconds)) {
/* Check 'common' datasets, if any */
if(ncommon > 0) {
@@ -186,6 +295,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, /* Check common dataset */
if(check_dataset(fid, verbose, sym_com[v]->name, &record, mem_sid) < 0)
return -1;
+ memset(&record, 0, sizeof(record));
} /* end for */
} /* end if */
@@ -200,17 +310,10 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, /* Check random dataset */
if(check_dataset(fid, verbose, sym_rand[v]->name, &record, mem_sid) < 0)
return -1;
+ memset(&record, 0, sizeof(record));
} /* end for */
} /* end if */
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Closing file\n");
-
- /* Close the file */
- if(H5Fclose(fid) < 0)
- return -1;
-
/* Sleep for the appropriate # of seconds */
sleep(poll_time);
@@ -218,6 +321,14 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, curr_time = time(NULL);
} /* end while */
+ /* Emit informational message */
+ if(verbose)
+ fprintf(stderr, "Closing file\n");
+
+ /* Close the file */
+ if(H5Fclose(fid) < 0)
+ return -1;
+
/* Close the memory dataspace */
if(H5Sclose(mem_sid) < 0)
return -1;
@@ -244,9 +355,17 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, static void
usage(void)
{
+ printf("\n");
printf("Usage error!\n");
- printf("Usage: swmr_reader [-q] [-s <# of seconds to sleep between polling>] [-h <# of common symbols to poll>] [-l <# of random symbols to poll>] [-r <random # seed>] <# of seconds to test>\n");
- printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'), 5 common symbols to poll ('-h 5') and 10 random symbols to poll ('-l 10')\n");
+ printf("\n");
+ printf("Usage: swmr_remove_reader [-q] [-s <# of seconds to sleep between\n");
+ printf(" polling>] [-h <# of common symbols to poll>] [-l <# of random symbols\n");
+ printf(" to poll>] [-r <random seed>] <# of seconds to test>\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given), 1 second between polling ('-s 1'),\n");
+ printf("5 common symbols to poll ('-h 5'), 10 random symbols to poll ('-l 10'),\n");
+ printf("and will generate a random seed (no -r given).\n");
+ printf("\n");
exit(1);
}
diff --git a/test/swmr_remove_writer.c b/test/swmr_remove_writer.c index 0a9cdbb..c05cf6f 100644 --- a/test/swmr_remove_writer.c +++ b/test/swmr_remove_writer.c @@ -13,12 +13,66 @@ * access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*-------------------------------------------------------------------------
+ *
+ * Created: swmr_remove_writer.c
+ *
+ * Purpose: Removes data from a randomly selected subset of the datasets
+ * in the SWMR test file.
+ *
+ * This program is intended to run concurrently with the
+ * swmr_remove_reader program. It is also run AFTER a sequential
+ * (not concurrent!) invoking of swmr_writer so the writer
+ * can dump a bunch of data into the datasets. Otherwise,
+ * there wouldn't be much to shrink :)
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include <assert.h>
#include <sys/time.h>
#include "swmr_common.h"
-#define MAX_REMOVE_SIZE 10
-
+/****************/
+/* Local Macros */
+/****************/
+
+/* The maximum number of records to remove in one step */
+#define MAX_REMOVE_SIZE 10
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+static hid_t open_skeleton(const char *filename, unsigned verbose);
+static int remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks,
+ unsigned long flush_count);
+static void usage(void);
+
+
+/*-------------------------------------------------------------------------
+ * Function: open_skeleton
+ *
+ * Purpose: Opens the SWMR HDF5 file and datasets.
+ *
+ * Parameters: const char *filename
+ * The filename of the SWMR HDF5 file to open
+ *
+ * unsigned verbose
+ * Whether or not to emit verbose console messages
+ *
+ * Return: Success: The file ID of the opened SWMR file
+ * The dataset IDs are stored in a global array
+ *
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static hid_t
open_skeleton(const char *filename, unsigned verbose)
{
@@ -28,6 +82,8 @@ open_skeleton(const char *filename, unsigned verbose) hsize_t dim[2]; /* Dataspace dimensions */
unsigned u, v; /* Local index variable */
+ assert(filename);
+
/* Create file access property list */
if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
return -1;
@@ -81,6 +137,30 @@ open_skeleton(const char *filename, unsigned verbose) return fid;
}
+
+/*-------------------------------------------------------------------------
+ * Function: remove_records
+ *
+ * Purpose: Removes a specified number of records from random datasets in
+ * the SWMR test file.
+ *
+ * Parameters: hid_t fid
+ * The file ID of the SWMR HDF5 file
+ *
+ * unsigned verbose
+ * Whether or not to emit verbose console messages
+ *
+ * unsigned long nshrinks
+ * # of records to remove from the datasets
+ *
+ * unsigned long flush_count
+ * # of records to write before flushing the file to disk
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static int
remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks, unsigned long flush_count)
{
@@ -88,6 +168,8 @@ remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks, unsigned lon hsize_t dim[2] = {1,0}; /* Dataspace dimensions */
unsigned long u, v; /* Local index variables */
+ assert(fid >= 0);
+
/* Remove records from random datasets, according to frequency distribution */
shrink_to_flush = flush_count;
for(u = 0; u < nshrinks; u++) {
@@ -140,10 +222,18 @@ remove_records(hid_t fid, unsigned verbose, unsigned long nshrinks, unsigned lon static void
usage(void)
{
+ printf("\n");
printf("Usage error!\n");
- printf("Usage: swmr_remove_writer [-q] [-f <# of shrinks between flushing file contents>] [-r <random # seed>] <# of shrinks>\n");
- printf("<# of shrinks between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of shrinks> - 1)\n");
- printf("Defaults to verbose (no '-q' given) and flushing every 1000 shrinks('-f 1000')\n");
+ printf("\n");
+ printf("Usage: swmr_remove_writer [-q] [-f <# of shrinks between flushing\n");
+ printf(" file contents>] [-r <random seed>] <# of shrinks>\n");
+ printf("\n");
+ printf("<# of shrinks between flushing file contents> should be 0 (for no\n");
+ printf("flushing) or between 1 and (<# of shrinks> - 1)\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given), flushing every 1000 shrinks\n");
+ printf("('-f 1000'), and will generate a random seed (no -r given).\n");
+ printf("\n");
exit(1);
}
diff --git a/test/swmr_sparse_reader.c b/test/swmr_sparse_reader.c index 052e8c5..1402435 100644 --- a/test/swmr_sparse_reader.c +++ b/test/swmr_sparse_reader.c @@ -13,20 +13,95 @@ * access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*-------------------------------------------------------------------------
+ *
+ * Created: swmr_sparse_reader.c
+ *
+ * Purpose: Reads data from a randomly selected subset of the datasets
+ * in the SWMR test file. Unlike the regular reader, these
+ * datasets will be shrinking.
+ *
+ * This program is intended to run concurrently with the
+ * swmr_sparse_writer program.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include <assert.h>
#include "swmr_common.h"
+
#include <unistd.h>
+/****************/
+/* Local Macros */
+/****************/
+
#define TIMEOUT 300
+/*******************/
+/* Local Variables */
+/*******************/
+
static hid_t symbol_tid = (-1);
+/********************/
+/* Local Prototypes */
+/********************/
+
+static int check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol,
+ symbol_t *record, hid_t rec_sid);
+static int read_records(const char *filename, unsigned verbose, unsigned long nrecords,
+ unsigned poll_time, unsigned reopen_count);
+static void usage(void);
+
+
+/*-------------------------------------------------------------------------
+ * Function: check_dataset
+ *
+ * Purpose: For a given dataset, checks to make sure that the stated
+ * and actual sizes are the same. If they are not, then
+ * we have an inconsistent dataset due to a SWMR error.
+ *
+ * Parameters: hid_t fid
+ * The SWMR test file's ID.
+ *
+ * unsigned verbose
+ * Whether verbose console output is desired.
+ *
+ * const symbol_info_t *symbol
+ * The dataset from which to read (the ID is in the struct).
+ * Must be pre-allocated.
+ *
+ * symbol_t *record
+ * Memory for the record. Must be pre-allocated.
+ *
+ * hid_t rec_sid
+ * The memory dataspace for access. It's always the same so
+ * there is no need to re-create it every time this function
+ * is called.
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static int
check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol, symbol_t *record,
hid_t rec_sid)
{
hid_t dsid; /* Dataset ID */
hid_t file_sid; /* Dataset's space ID */
- hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
+ hsize_t start[2] = {0, 0}; /* Hyperslab selection values */
+ hsize_t count[2] = {1, 1}; /* Hyperslab selection values */
+
+ assert(fid >= 0);
+ assert(symbol);
+ assert(record);
+ assert(rec_sid >= 0);
/* Open dataset for symbol */
if((dsid = H5Dopen2(fid, symbol->name, H5P_DEFAULT)) < 0)
@@ -83,6 +158,34 @@ check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol, symbol_t return 0;
} /* end check_dataset() */
+
+/*-------------------------------------------------------------------------
+ * Function: read_records
+ *
+ * Purpose: For a given dataset, checks to make sure that the stated
+ * and actual sizes are the same. If they are not, then
+ * we have an inconsistent dataset due to a SWMR error.
+ *
+ * Parameters: const char *filename
+ * The SWMR test file's name.
+ *
+ * unsigned verbose
+ * Whether verbose console output is desired.
+ *
+ * unsigned long nrecords
+ * The total number of records to read.
+ *
+ * unsigned poll_time
+ * The amount of time to sleep (s).
+ *
+ * unsigned reopen_count
+ *
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static int
read_records(const char *filename, unsigned verbose, unsigned long nrecords,
unsigned poll_time, unsigned reopen_count)
@@ -96,6 +199,10 @@ read_records(const char *filename, unsigned verbose, unsigned long nrecords, unsigned iter_to_reopen = reopen_count; /* # of iterations until reopen */
unsigned long u; /* Local index variable */
hid_t fapl;
+
+ assert(filename);
+ assert(poll_time != 0);
+
fapl = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fclose_degree(fapl, H5F_CLOSE_SEMI);
/* Emit informational message */
@@ -188,6 +295,7 @@ read_records(const char *filename, unsigned verbose, unsigned long nrecords, /* Check dataset */
if(check_dataset(fid, verbose, symbol, &record, mem_sid) < 0)
return -1;
+ memset(&record, 0, sizeof(record));
/* Check for reopen */
iter_to_reopen--;
@@ -219,10 +327,18 @@ read_records(const char *filename, unsigned verbose, unsigned long nrecords, static void
usage(void)
{
+ printf("\n");
printf("Usage error!\n");
- printf("Usage: swmr_sparse_reader [-q] [-s <# of seconds to wait for writer>] [-n <# of reads between reopens>] <# of records>\n");
- printf("Defaults to verbose (no '-q' given), 1 second wait ('-s 1') and 1 read between reopens ('-r 1')\n");
- printf("Note that the # of records *must* be the same as that supplied to swmr_sparse_writer\n");
+ printf("\n");
+ printf("Usage: swmr_sparse_reader [-q] [-s <# of seconds to wait for writer>]\n");
+ printf(" [-n <# of reads between reopens>] <# of records>\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given), 1 second wait ('-s 1') and 1 read\n");
+ printf("between reopens ('-r 1')\n");
+ printf("\n");
+ printf("Note that the # of records *must* be the same as that supplied to\n");
+ printf("swmr_sparse_writer\n");
+ printf("\n");
exit(1);
} /* end usage() */
@@ -330,4 +446,3 @@ int main(int argc, const char *argv[]) return 0;
}
-
diff --git a/test/swmr_sparse_writer.c b/test/swmr_sparse_writer.c index 74dbb13..00a0256 100644 --- a/test/swmr_sparse_writer.c +++ b/test/swmr_sparse_writer.c @@ -13,10 +13,62 @@ * access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+ /*-------------------------------------------------------------------------
+ *
+ * Created: swmr_sparse_writer.c
+ *
+ * Purpose: Writes data to a randomly selected subset of the datasets
+ * in the SWMR test file.
+ *
+ * This program is intended to run concurrently with the
+ * swmr_sparse_reader program.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/***********/
+/* Headers */
+/***********/
+
+#include <assert.h>
+
#include "swmr_common.h"
+/****************/
+/* Local Macros */
+/****************/
+
#define BUSY_WAIT 100000
+/********************/
+/* Local Prototypes */
+/********************/
+
+static hid_t open_skeleton(const char *filename, unsigned verbose);
+static int add_records(hid_t fid, unsigned verbose, unsigned long nrecords,
+ unsigned long flush_count);
+static void usage(void);
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: open_skeleton
+ *
+ * Purpose: Opens the SWMR HDF5 file and datasets.
+ *
+ * Parameters: const char *filename
+ * The filename of the SWMR HDF5 file to open
+ *
+ * unsigned verbose
+ * Whether or not to emit verbose console messages
+ *
+ * Return: Success: The file ID of the opened SWMR file
+ * The dataset IDs are stored in a global array
+ *
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static hid_t
open_skeleton(const char *filename, unsigned verbose)
{
@@ -26,6 +78,8 @@ open_skeleton(const char *filename, unsigned verbose) unsigned seed; /* Seed for random number generator */
unsigned u, v; /* Local index variable */
+ assert(filename);
+
/* Create file access property list */
if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
return -1;
@@ -82,12 +136,37 @@ open_skeleton(const char *filename, unsigned verbose) return fid;
}
+
+/*-------------------------------------------------------------------------
+ * Function: add_records
+ *
+ * Purpose: Writes a specified number of records to random datasets in
+ * the SWMR test file.
+ *
+ * Parameters: hid_t fid
+ * The file ID of the SWMR HDF5 file
+ *
+ * unsigned verbose
+ * Whether or not to emit verbose console messages
+ *
+ * unsigned long nrecords
+ * # of records to write to the datasets
+ *
+ * unsigned long flush_count
+ * # of records to write before flushing the file to disk
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
static int
add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long flush_count)
{
hid_t tid; /* Datatype ID for records */
hid_t mem_sid; /* Memory dataspace ID */
- hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
+ hsize_t start[2] = {0, 0}; /* Hyperslab selection values */
+ hsize_t count[2] = {1, 1}; /* Hyperslab selection values */
symbol_t record; /* The record to add to the dataset */
H5AC_cache_config_t mdc_config_orig; /* Original metadata cache configuration */
H5AC_cache_config_t mdc_config_cork; /* Corked metadata cache configuration */
@@ -96,6 +175,8 @@ add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long f hsize_t dim[2] = {1,0}; /* Dataspace dimensions */
unsigned long u, v; /* Local index variables */
+ assert(fid >= 0);
+
/* Reset the record */
/* (record's 'info' field might need to change for each record written, also) */
memset(&record, 0, sizeof(record));
@@ -240,10 +321,18 @@ add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long f static void
usage(void)
{
+ printf("\n");
printf("Usage error!\n");
- printf("Usage: swmr_sparse_writer [-q] [-f <# of records to write between flushing file contents>] <# of records>\n");
- printf("<# of records to write between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of records> - 1)\n");
- printf("Defaults to verbose (no '-q' given) and flushing every 1000 records('-f 1000')\n");
+ printf("\n");
+ printf("Usage: swmr_sparse_writer [-q] [-f <# of records to write between\n");
+ printf(" flushing file contents>] <# of records>\n");
+ printf("\n");
+ printf("<# of records to write between flushing file contents> should be 0\n");
+ printf("(for no flushing) or between 1 and (<# of records> - 1)\n");
+ printf("\n");
+ printf("Defaults to verbose (no '-q' given) and flushing every 1000 records\n");
+ printf("('-f 1000')\n");
+ printf("\n");
exit(1);
}
diff --git a/test/swmr_writer.c b/test/swmr_writer.c index f4e33e5..8ff7312 100644 --- a/test/swmr_writer.c +++ b/test/swmr_writer.c @@ -1,334 +1,418 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * Copyright by The HDF Group. *
- * Copyright by the Board of Trustees of the University of Illinois. *
- * All rights reserved. *
- * *
- * This file is part of HDF5. The full HDF5 copyright notice, including *
- * terms governing use, modification, and redistribution, is contained in *
- * the files COPYING and Copyright.html. COPYING can be found at the root *
- * of the source code distribution tree; Copyright.html can be found at the *
- * root level of an installed copy of the electronic HDF5 document set and *
- * is linked from the top-level documents page. It can also be found at *
- * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
- * access to either file, you may request a copy from help@hdfgroup.org. *
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-#include <sys/time.h>
-
-#include "swmr_common.h"
-
-static hid_t
-open_skeleton(const char *filename, unsigned verbose)
-{
- hid_t fid; /* File ID for new HDF5 file */
- hid_t fapl; /* File access property list */
- unsigned u, v; /* Local index variable */
-
- /* Create file access property list */
- if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
- return -1;
-
-#ifdef QAK
- /* Increase the initial size of the metadata cache */
- {
- H5AC_cache_config_t mdc_config;
-
- mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
- H5Pget_mdc_config(fapl, &mdc_config);
- fprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size);
- fprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length);
- mdc_config.set_initial_size = 1;
- mdc_config.initial_size = 16 * 1024 * 1024;
- /* mdc_config.epoch_length = 5000; */
- H5Pset_mdc_config(fapl, &mdc_config);
- }
-#endif /* QAK */
-
-#ifdef QAK
- H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024));
-#endif /* QAK */
-
- /* Open the file */
- if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0)
- return -1;
-
- /* Close file access property list */
- if(H5Pclose(fapl) < 0)
- return -1;
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Opening datasets\n");
-
- /* Open the datasets */
- for(u = 0; u < NLEVELS; u++)
- for(v = 0; v < symbol_count[u]; v++) {
- if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0)
- return -1;
- symbol_info[u][v].nrecords = 0;
- } /* end for */
-
- return(fid);
-}
-
-static int
-add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long flush_count)
-{
- hid_t tid; /* Datatype ID for records */
- hid_t mem_sid; /* Memory dataspace ID */
- hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */
- hsize_t dim[2] = {1, 0}; /* Dataspace dimensions */
- symbol_t record; /* The record to add to the dataset */
- H5AC_cache_config_t mdc_config_orig; /* Original metadata cache configuration */
- H5AC_cache_config_t mdc_config_cork; /* Corked metadata cache configuration */
- unsigned long rec_to_flush; /* # of records left to write before flush */
- unsigned long u, v; /* Local index variables */
-
- /* Reset the record */
- /* (record's 'info' field might need to change for each record written, also) */
- memset(&record, 0, sizeof(record));
-
- /* Create a dataspace for the record to add */
- if((mem_sid = H5Screate(H5S_SCALAR)) < 0)
- return -1;
-
- /* Create datatype for appending records */
- if((tid = create_symbol_datatype()) < 0)
- return -1;
-
- /* Get the current metadata cache configuration, and set up the corked
- * configuration */
- mdc_config_orig.version = H5AC__CURR_CACHE_CONFIG_VERSION;
- if(H5Fget_mdc_config(fid, &mdc_config_orig) < 0)
- return -1;
- memcpy(&mdc_config_cork, &mdc_config_orig, sizeof(mdc_config_cork));
- mdc_config_cork.evictions_enabled = FALSE;
- mdc_config_cork.incr_mode = H5C_incr__off;
- mdc_config_cork.flash_incr_mode = H5C_flash_incr__off;
- mdc_config_cork.decr_mode = H5C_decr__off;
-
- /* Add records to random datasets, according to frequency distribution */
- rec_to_flush = flush_count;
- for(u = 0; u < nrecords; u++) {
- symbol_info_t *symbol; /* Symbol to write record to */
- hid_t file_sid; /* Dataset's space ID */
-
- /* Get a random dataset, according to the symbol distribution */
- symbol = choose_dataset();
-
- /* Set the record's ID (equal to its position) */
- record.rec_id = symbol->nrecords;
-
- /* Get the coordinate to write */
- start[1] = symbol->nrecords;
-
- /* Cork the metadata cache, to prevent the object header from being
- * flushed before the data has been written */
- /*if(H5Fset_mdc_config(fid, &mdc_config_cork) < 0)
- return(-1);*/
-
- /* Extend the dataset's dataspace to hold the new record */
- symbol->nrecords++;
- dim[1] = symbol->nrecords;
- if(H5Dset_extent(symbol->dsid, dim) < 0)
- return -1;
-
- /* Get the dataset's dataspace */
- if((file_sid = H5Dget_space(symbol->dsid)) < 0)
- return -1;
-
- /* Choose the last record in the dataset */
- if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0)
- return -1;
-
- /* Write record to the dataset */
- if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0)
- return -1;
-
- /* Uncork the metadata cache */
- /*if(H5Fset_mdc_config(fid, &mdc_config_orig) < 0)
- return -1;*/
-
- /* Close the dataset's dataspace */
- if(H5Sclose(file_sid) < 0)
- return -1;
-
- /* Check for flushing file */
- if(flush_count > 0) {
- /* Decrement count of records to write before flushing */
- rec_to_flush--;
-
- /* Check for counter being reached */
- if(0 == rec_to_flush) {
- /* Flush contents of file */
- if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
- return -1;
-
- /* Reset flush counter */
- rec_to_flush = flush_count;
- } /* end if */
- } /* end if */
- } /* end for */
-
- /* Close the memory dataspace */
- if(H5Sclose(mem_sid) < 0)
- return -1;
-
- /* Close the datatype */
- if(H5Tclose(tid) < 0)
- return -1;
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Closing datasets\n");
-
- /* Close the datasets */
- for(u = 0; u < NLEVELS; u++)
- for(v = 0; v < symbol_count[u]; v++)
- if(H5Dclose(symbol_info[u][v].dsid) < 0)
- return -1;
-
- return 0;
-}
-
-static void
-usage(void)
-{
- printf("Usage error!\n");
- printf("Usage: swmr_writer [-q] [-f <# of records to write between flushing file contents>] [-r <random # seed>] <# of records>\n");
- printf("<# of records to write between flushing file contents> should be 0 (for no flushing) or between 1 and (<# of records> - 1)\n");
- printf("Defaults to verbose (no '-q' given) and flushing every 10000 records('-f 10000')\n");
- exit(1);
-}
-
-int main(int argc, const char *argv[])
-{
- hid_t fid; /* File ID for file opened */
- long nrecords = 0; /* # of records to append */
- long flush_count = 10000; /* # of records to write between flushing file */
- unsigned verbose = 1; /* Whether to emit some informational messages */
- unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */
- unsigned random_seed = 0; /* Random # seed */
- unsigned u; /* Local index variable */
- int temp;
-
- /* Parse command line options */
- if(argc < 2)
- usage();
- if(argc > 1) {
- u = 1;
- while(u < (unsigned)argc) {
- if(argv[u][0] == '-') {
- switch(argv[u][1]) {
- /* # of records to write between flushing file */
- case 'f':
- flush_count = atol(argv[u + 1]);
- if(flush_count < 0)
- usage();
- u += 2;
- break;
-
- /* Be quiet */
- case 'q':
- verbose = 0;
- u++;
- break;
-
- /* Random # seed */
- case 'r':
- use_seed = 1;
- temp = atoi(argv[u + 1]);
- if(temp < 0)
- usage();
- else
- random_seed = (unsigned)temp;
- u += 2;
- break;
-
- default:
- usage();
- break;
- } /* end switch */
- } /* end if */
- else {
- /* Get the number of records to append */
- nrecords = atol(argv[u]);
- if(nrecords <= 0)
- usage();
-
- u++;
- } /* end else */
- } /* end while */
- } /* end if */
- if(nrecords <= 0)
- usage();
- if(flush_count >= nrecords)
- usage();
-
- /* Emit informational message */
- if(verbose) {
- fprintf(stderr, "Parameters:\n");
- fprintf(stderr, "\t# of records between flushes = %ld\n", flush_count);
- fprintf(stderr, "\t# of records to write = %ld\n", nrecords);
- } /* end if */
-
- /* Set the random seed */
- if(0 == use_seed) {
- struct timeval t;
- gettimeofday(&t, NULL);
- random_seed = (unsigned)((t.tv_sec * 1000) + t.tv_usec);
- } /* end if */
- srandom(random_seed);
- /* ALWAYS emit the random seed for possible debugging */
- fprintf(stderr, "Using writer random seed: %u\n", random_seed);
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Generating symbol names\n");
-
- /* Generate dataset names */
- if(generate_symbols() < 0)
- return -1;
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Opening skeleton file: %s\n", FILENAME);
-
- /* Open file skeleton */
- if((fid = open_skeleton(FILENAME, verbose)) < 0) {
- fprintf(stderr, "Error opening skeleton file!\n");
- exit(1);
- } /* end if */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Adding records\n");
-
- /* Append records to datasets */
- if(add_records(fid, verbose, (unsigned long)nrecords, (unsigned long)flush_count) < 0) {
- fprintf(stderr, "Error appending records to datasets!\n");
- exit(1);
- } /* end if */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Releasing symbols\n");
-
- /* Clean up the symbols */
- if(shutdown_symbols() < 0) {
- fprintf(stderr, "Error releasing symbols!\n");
- exit(1);
- } /* end if */
-
- /* Emit informational message */
- if(verbose)
- fprintf(stderr, "Closing objects\n");
-
- /* Close objects opened */
- if(H5Fclose(fid) < 0) {
- fprintf(stderr, "Error closing file!\n");
- exit(1);
- } /* end if */
-
- return 0;
-}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: swmr_writer.c + * + * Purpose: Writes data to a randomly selected subset of the datasets + * in the SWMR test file. + * + * This program is intended to run concurrently with the + * swmr_reader program. + * + *------------------------------------------------------------------------- + */ + +/***********/ +/* Headers */ +/***********/ + +#include <assert.h> +#include <sys/time.h> + +#include "swmr_common.h" + +/********************/ +/* Local Prototypes */ +/********************/ + +static hid_t open_skeleton(const char *filename, unsigned verbose); +static int add_records(hid_t fid, unsigned verbose, unsigned long nrecords, + unsigned long flush_count); +static void usage(void); + + +/*------------------------------------------------------------------------- + * Function: open_skeleton + * + * Purpose: Opens the SWMR HDF5 file and datasets. + * + * Parameters: const char *filename + * The filename of the SWMR HDF5 file to open + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * Return: Success: The file ID of the opened SWMR file + * The dataset IDs are stored in a global array + * + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static hid_t +open_skeleton(const char *filename, unsigned verbose) +{ + hid_t fid; /* File ID for new HDF5 file */ + hid_t fapl; /* File access property list */ + unsigned u, v; /* Local index variable */ + + assert(filename); + + /* Create file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return -1; + +#ifdef QAK + /* Increase the initial size of the metadata cache */ + { + H5AC_cache_config_t mdc_config; + + mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + H5Pget_mdc_config(fapl, &mdc_config); + fprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); + fprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); + mdc_config.set_initial_size = 1; + mdc_config.initial_size = 16 * 1024 * 1024; + /* mdc_config.epoch_length = 5000; */ + H5Pset_mdc_config(fapl, &mdc_config); + } +#endif /* QAK */ + +#ifdef QAK + H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); +#endif /* QAK */ + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) + return -1; + + /* Close file access property list */ + if(H5Pclose(fapl) < 0) + return -1; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Opening datasets\n"); + + /* Open the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) { + if((symbol_info[u][v].dsid = H5Dopen2(fid, symbol_info[u][v].name, H5P_DEFAULT)) < 0) + return -1; + symbol_info[u][v].nrecords = 0; + } /* end for */ + + return(fid); +} + + +/*------------------------------------------------------------------------- + * Function: add_records + * + * Purpose: Writes a specified number of records to random datasets in + * the SWMR test file. + * + * Parameters: hid_t fid + * The file ID of the SWMR HDF5 file + * + * unsigned verbose + * Whether or not to emit verbose console messages + * + * unsigned long nrecords + * # of records to write to the datasets + * + * unsigned long flush_count + * # of records to write before flushing the file to disk + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static int +add_records(hid_t fid, unsigned verbose, unsigned long nrecords, unsigned long flush_count) +{ + hid_t tid; /* Datatype ID for records */ + hid_t mem_sid; /* Memory dataspace ID */ + hsize_t start[2] = {0, 0}, count[2] = {1, 1}; /* Hyperslab selection values */ + hsize_t dim[2] = {1, 0}; /* Dataspace dimensions */ + symbol_t record; /* The record to add to the dataset */ + H5AC_cache_config_t mdc_config_orig; /* Original metadata cache configuration */ + H5AC_cache_config_t mdc_config_cork; /* Corked metadata cache configuration */ + unsigned long rec_to_flush; /* # of records left to write before flush */ + unsigned long u, v; /* Local index variables */ + + assert(fid >= 0); + + /* Reset the record */ + /* (record's 'info' field might need to change for each record written, also) */ + memset(&record, 0, sizeof(record)); + + /* Create a dataspace for the record to add */ + if((mem_sid = H5Screate(H5S_SCALAR)) < 0) + return -1; + + /* Create datatype for appending records */ + if((tid = create_symbol_datatype()) < 0) + return -1; + + /* Get the current metadata cache configuration, and set up the corked + * configuration */ + mdc_config_orig.version = H5AC__CURR_CACHE_CONFIG_VERSION; + if(H5Fget_mdc_config(fid, &mdc_config_orig) < 0) + return -1; + memcpy(&mdc_config_cork, &mdc_config_orig, sizeof(mdc_config_cork)); + mdc_config_cork.evictions_enabled = FALSE; + mdc_config_cork.incr_mode = H5C_incr__off; + mdc_config_cork.flash_incr_mode = H5C_flash_incr__off; + mdc_config_cork.decr_mode = H5C_decr__off; + + /* Add records to random datasets, according to frequency distribution */ + rec_to_flush = flush_count; + for(u = 0; u < nrecords; u++) { + symbol_info_t *symbol; /* Symbol to write record to */ + hid_t file_sid; /* Dataset's space ID */ + + /* Get a random dataset, according to the symbol distribution */ + symbol = choose_dataset(); + + /* Set the record's ID (equal to its position) */ + record.rec_id = symbol->nrecords; + + /* Get the coordinate to write */ + start[1] = symbol->nrecords; + + /* Cork the metadata cache, to prevent the object header from being + * flushed before the data has been written */ + /*if(H5Fset_mdc_config(fid, &mdc_config_cork) < 0) + return(-1);*/ + + /* Extend the dataset's dataspace to hold the new record */ + symbol->nrecords++; + dim[1] = symbol->nrecords; + if(H5Dset_extent(symbol->dsid, dim) < 0) + return -1; + + /* Get the dataset's dataspace */ + if((file_sid = H5Dget_space(symbol->dsid)) < 0) + return -1; + + /* Choose the last record in the dataset */ + if(H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) + return -1; + + /* Write record to the dataset */ + if(H5Dwrite(symbol->dsid, tid, mem_sid, file_sid, H5P_DEFAULT, &record) < 0) + return -1; + + /* Uncork the metadata cache */ + /*if(H5Fset_mdc_config(fid, &mdc_config_orig) < 0) + return -1;*/ + + /* Close the dataset's dataspace */ + if(H5Sclose(file_sid) < 0) + return -1; + + /* Check for flushing file */ + if(flush_count > 0) { + /* Decrement count of records to write before flushing */ + rec_to_flush--; + + /* Check for counter being reached */ + if(0 == rec_to_flush) { + /* Flush contents of file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + return -1; + + /* Reset flush counter */ + rec_to_flush = flush_count; + } /* end if */ + } /* end if */ + } /* end for */ + + /* Close the memory dataspace */ + if(H5Sclose(mem_sid) < 0) + return -1; + + /* Close the datatype */ + if(H5Tclose(tid) < 0) + return -1; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Closing datasets\n"); + + /* Close the datasets */ + for(u = 0; u < NLEVELS; u++) + for(v = 0; v < symbol_count[u]; v++) + if(H5Dclose(symbol_info[u][v].dsid) < 0) + return -1; + + return 0; +} + +static void +usage(void) +{ + printf("\n"); + printf("Usage error!\n"); + printf("\n"); + printf("Usage: swmr_writer [-q] [-f <# of records to write between flushing\n"); + printf(" file contents>] [-r <random seed>] <# of records>\n"); + printf("\n"); + printf("<# of records to write between flushing file contents> should be 0\n"); + printf("(for no flushing) or between 1 and (<# of records> - 1).\n"); + printf("\n"); + printf("<# of records> must be specified.\n"); + printf("\n"); + printf("Defaults to verbose (no '-q' given), flushing every 10000 records\n"); + printf("('-f 10000'), and will generate a random seed (no -r given).\n"); + printf("\n"); + exit(1); +} + +int main(int argc, const char *argv[]) +{ + hid_t fid; /* File ID for file opened */ + long nrecords = 0; /* # of records to append */ + long flush_count = 10000; /* # of records to write between flushing file */ + unsigned verbose = 1; /* Whether to emit some informational messages */ + unsigned use_seed = 0; /* Set to 1 if a seed was set on the command line */ + unsigned random_seed = 0; /* Random # seed */ + unsigned u; /* Local index variable */ + int temp; + + /* Parse command line options */ + if(argc < 2) + usage(); + if(argc > 1) { + u = 1; + while(u < (unsigned)argc) { + if(argv[u][0] == '-') { + switch(argv[u][1]) { + /* # of records to write between flushing file */ + case 'f': + flush_count = atol(argv[u + 1]); + if(flush_count < 0) + usage(); + u += 2; + break; + + /* Be quiet */ + case 'q': + verbose = 0; + u++; + break; + + /* Random # seed */ + case 'r': + use_seed = 1; + temp = atoi(argv[u + 1]); + if(temp < 0) + usage(); + else + random_seed = (unsigned)temp; + u += 2; + break; + + default: + usage(); + break; + } /* end switch */ + } /* end if */ + else { + /* Get the number of records to append */ + nrecords = atol(argv[u]); + if(nrecords <= 0) + usage(); + + u++; + } /* end else */ + } /* end while */ + } /* end if */ + if(nrecords <= 0) + usage(); + if(flush_count >= nrecords) + usage(); + + /* Emit informational message */ + if(verbose) { + fprintf(stderr, "Parameters:\n"); + fprintf(stderr, "\t# of records between flushes = %ld\n", flush_count); + fprintf(stderr, "\t# of records to write = %ld\n", nrecords); + } /* end if */ + + /* Set the random seed */ + if(0 == use_seed) { + struct timeval t; + gettimeofday(&t, NULL); + random_seed = (unsigned)((t.tv_sec * 1000) + t.tv_usec); + } /* end if */ + srandom(random_seed); + /* ALWAYS emit the random seed for possible debugging */ + fprintf(stderr, "Using writer random seed: %u\n", random_seed); + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Generating symbol names\n"); + + /* Generate dataset names */ + if(generate_symbols() < 0) + return -1; + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Opening skeleton file: %s\n", FILENAME); + + /* Open file skeleton */ + if((fid = open_skeleton(FILENAME, verbose)) < 0) { + fprintf(stderr, "Error opening skeleton file!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Adding records\n"); + + /* Append records to datasets */ + if(add_records(fid, verbose, (unsigned long)nrecords, (unsigned long)flush_count) < 0) { + fprintf(stderr, "Error appending records to datasets!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Releasing symbols\n"); + + /* Clean up the symbols */ + if(shutdown_symbols() < 0) { + fprintf(stderr, "Error releasing symbols!\n"); + exit(1); + } /* end if */ + + /* Emit informational message */ + if(verbose) + fprintf(stderr, "Closing objects\n"); + + /* Close objects opened */ + if(H5Fclose(fid) < 0) { + fprintf(stderr, "Error closing file!\n"); + exit(1); + } /* end if */ + + return 0; +} diff --git a/test/testswmr.sh b/test/testswmr.sh index d13c6ed..43f9f39 100755 --- a/test/testswmr.sh +++ b/test/testswmr.sh @@ -72,7 +72,7 @@ done for index_type in "-i b1" "-i ea" "-i b2" do # Try with and without compression - for compress in "" "-c 1" + for compress in "" "-c 5" do echo echo "###############################################################################" |