summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlbert Cheng <acheng@hdfgroup.org>2002-05-13 19:55:33 (GMT)
committerAlbert Cheng <acheng@hdfgroup.org>2002-05-13 19:55:33 (GMT)
commitd07e0dd9a3d98981ba49c1a39998faf7cb522075 (patch)
treef9d03b17c9c1b722c8145cd3b7cdbc8f2cb4386b
parent4793f81ae10d65a681ce440ce072b510b1ec8d92 (diff)
downloadhdf5-d07e0dd9a3d98981ba49c1a39998faf7cb522075.zip
hdf5-d07e0dd9a3d98981ba49c1a39998faf7cb522075.tar.gz
hdf5-d07e0dd9a3d98981ba49c1a39998faf7cb522075.tar.bz2
[svn-r5407] Purpose:
Bug fix Description: Was not able to handle data size (file size) larger than 32bits. Was using long, which is only 4 bytes big in SP, thus overflowing into negative when trying to address 2GB or larger. Solution: Changed those variables involved in file size/offset calculation to type off_t. (If a certain system/compiler has off_t defined as 4 bytes, it can't write to file size larger than 2GB anyway.) Note that the lseek of SP with -D_LARGE_FILE still fails for offset larger than 2GB (works for 2GB). That has to be fixed soon. Platforms tested: burrwhite (linux 2.4) and modi4 parallel.
-rw-r--r--perform/pio_engine.c27
-rw-r--r--perform/pio_perf.c31
-rw-r--r--perform/pio_perf.h2
3 files changed, 34 insertions, 26 deletions
diff --git a/perform/pio_engine.c b/perform/pio_engine.c
index e48129a..abc0668 100644
--- a/perform/pio_engine.c
+++ b/perform/pio_engine.c
@@ -129,9 +129,9 @@ typedef union _file_descr {
static char *pio_create_filename(iotype iot, const char *base_name,
char *fullname, size_t size);
static herr_t do_write(file_descr *fd, iotype iot, long ndsets,
- long nelmts, long buf_size, void *buffer);
+ long nelmts, size_t buf_size, void *buffer);
static herr_t do_read(file_descr *fd, iotype iot, long ndsets,
- long nelmts, long buf_size, void *buffer /*out*/);
+ long nelmts, size_t buf_size, void *buffer /*out*/);
static herr_t do_fopen(iotype iot, char *fname, file_descr *fd /*out*/,
int flags);
static herr_t do_fclose(iotype iot, file_descr *fd);
@@ -157,9 +157,10 @@ do_pio(parameters param)
char fname[FILENAME_MAX];
int maxprocs;
int nfiles, nf;
- long ndsets, nelmts;
+ long ndsets;
+ long nelmts;
char *buffer = NULL; /*data buffer pointer */
- long buf_size; /*data buffer size in bytes */
+ size_t buf_size; /*data buffer size in bytes */
/* HDF5 variables */
herr_t hrc; /*HDF5 return code */
@@ -244,7 +245,7 @@ buf_size=MIN(1024*1024, buf_size);
#endif
/* allocate data buffer */
- buffer = malloc((size_t)buf_size);
+ buffer = malloc(buf_size);
if (buffer == NULL){
fprintf(stderr, "malloc for data buffer size (%ld) failed\n",
@@ -468,7 +469,7 @@ pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t si
*/
static herr_t
do_write(file_descr *fd, iotype iot, long ndsets,
- long nelmts, long buf_size, void *buffer)
+ long nelmts, size_t buf_size, void *buffer)
{
int ret_code = SUCCESS;
int rc; /*routine return code */
@@ -480,7 +481,7 @@ do_write(file_descr *fd, iotype iot, long ndsets,
char dname[64];
off_t dset_offset; /*dataset offset in a file */
off_t file_offset; /*file offset of the next transfer */
- long dset_size; /*one dataset size in bytes */
+ off_t dset_size; /*one dataset size in bytes */
long nelmts_in_buf;
long elmts_begin; /*first elmt this process transfer */
long elmts_count; /*number of elmts this process transfer */
@@ -587,7 +588,9 @@ fprintf(stderr, "proc %d: elmts_begin=%ld, elmts_count=%ld\n",
/* Calculate offset of write within a dataset/file */
switch (iot) {
case RAWIO:
- file_offset = dset_offset + (elmts_begin + nelmts_written)*ELMT_SIZE;
+ /* need to (off_t) the elmnts_begin expression because they */
+ /* may be of smaller sized integer types */
+ file_offset = dset_offset + (off_t)(elmts_begin + nelmts_written)*ELMT_SIZE;
#if AKCDEBUG
fprintf(stderr, "proc %d: writes %ld bytes at file-offset %ld\n",
@@ -692,7 +695,7 @@ done:
*/
static herr_t
do_read(file_descr *fd, iotype iot, long ndsets,
- long nelmts, long buf_size, void *buffer /*out*/)
+ long nelmts, size_t buf_size, void *buffer /*out*/)
{
int ret_code = SUCCESS;
int rc; /*routine return code */
@@ -704,7 +707,7 @@ do_read(file_descr *fd, iotype iot, long ndsets,
char dname[64];
off_t dset_offset; /*dataset offset in a file */
off_t file_offset; /*file offset of the next transfer */
- long dset_size; /*one dataset size in bytes */
+ off_t dset_size; /*one dataset size in bytes */
long nelmts_in_buf;
long elmts_begin; /*first elmt this process transfer */
long elmts_count; /*number of elmts this process transfer */
@@ -797,7 +800,9 @@ fprintf(stderr, "proc %d: elmts_begin=%ld, elmts_count=%ld\n",
/* Calculate offset of read within a dataset/file */
switch (iot){
case RAWIO:
- file_offset = dset_offset + (elmts_begin + nelmts_read)*ELMT_SIZE;
+ /* need to (off_t) the elmnts_begin expression because they */
+ /* may be of smaller sized integer types */
+ file_offset = dset_offset + (off_t)(elmts_begin + nelmts_read)*ELMT_SIZE;
#if AKCDEBUG
fprintf(stderr, "proc %d: read %ld bytes at file-offset %ld\n",
diff --git a/perform/pio_perf.c b/perform/pio_perf.c
index 3afc58d..f9cac10 100644
--- a/perform/pio_perf.c
+++ b/perform/pio_perf.c
@@ -203,14 +203,14 @@ static struct long_options l_opts[] = {
struct options {
long io_types; /* bitmask of which I/O types to test */
const char *output_file; /* file to print report to */
- long file_size; /* size of file */
+ off_t file_size; /* size of file */
long num_dsets; /* number of datasets */
long num_files; /* number of files */
long num_iters; /* number of iterations */
long max_num_procs; /* maximum number of processes to use */
long min_num_procs; /* minimum number of processes to use */
- long max_xfer_size; /* maximum transfer buffer size */
- long min_xfer_size; /* minimum transfer buffer size */
+ size_t max_xfer_size; /* maximum transfer buffer size */
+ size_t min_xfer_size; /* minimum transfer buffer size */
};
typedef struct _minmax {
@@ -221,13 +221,13 @@ typedef struct _minmax {
} minmax;
/* local functions */
-static long parse_size_directive(const char *size);
+static off_t parse_size_directive(const char *size);
static struct options *parse_command_line(int argc, char *argv[]);
static void run_test_loop(struct options *options);
static int run_test(iotype iot, parameters parms);
static void output_all_info(minmax *mm, int count, int indent_level);
static void get_minmax(minmax *mm, double val);
-static minmax accumulate_minmax_stuff(minmax *mm, long raw_size, int count);
+static minmax accumulate_minmax_stuff(minmax *mm, off_t raw_size, int count);
static int create_comm_world(int num_procs, int *doing_pio);
static int destroy_comm_world(void);
static void output_report(const char *fmt, ...);
@@ -296,7 +296,6 @@ main(int argc, char **argv)
goto finish;
}
}
-
run_test_loop(opts);
finish:
@@ -355,7 +354,7 @@ run_test_loop(struct options *opts)
/* if performance needs restart, fewer processes may be needed. */
for (num_procs = opts->max_num_procs;
num_procs >= opts->min_num_procs; num_procs >>= 1) {
- register long buf_size;
+ register size_t buf_size;
parms.num_procs = num_procs;
@@ -411,7 +410,7 @@ run_test(iotype iot, parameters parms)
results res;
register int i, ret_value = SUCCESS;
int comm_size;
- long raw_size;
+ off_t raw_size;
minmax total_mm;
minmax *write_mpi_mm_table;
minmax *write_mm_table;
@@ -484,7 +483,7 @@ run_test(iotype iot, parameters parms)
read_gross_mm_table[i].num = 0;
}
- /* call Albert's testing here */
+ /* Do IO iteration times, collecting statics each time */
for (i = 0; i < parms.num_iters; ++i) {
double t;
@@ -529,6 +528,9 @@ run_test(iotype iot, parameters parms)
pio_time_destroy(res.timers);
}
+ /*
+ * Show various statics
+ */
/* show mpi write statics */
if (pio_debug_level >= 3) {
/* output all of the times for all iterations */
@@ -691,7 +693,7 @@ get_minmax(minmax *mm, double val)
* Modifications:
*/
static minmax
-accumulate_minmax_stuff(minmax *mm, long raw_size, int count)
+accumulate_minmax_stuff(minmax *mm, off_t raw_size, int count)
{
register int i;
minmax total_mm;
@@ -944,15 +946,16 @@ parse_command_line(int argc, char *argv[])
* M, m - Megabyte
* G, g - Gigabyte
*
- * Return: The size as a LONG. If an unknown size indicator is used, then
- * the program will exit with EXIT_FAILURE as the return value.
+ * Return: The size as a off_t because this is related to file size.
+ * If an unknown size indicator is used, then the program will
+ * exit with EXIT_FAILURE as the return value.
* Programmer: Bill Wendling, 18. December 2001
* Modifications:
*/
-static long
+static off_t
parse_size_directive(const char *size)
{
- long s;
+ off_t s;
char *endptr;
s = strtol(size, &endptr, 10);
diff --git a/perform/pio_perf.h b/perform/pio_perf.h
index 12811e3..d14c419 100644
--- a/perform/pio_perf.h
+++ b/perform/pio_perf.h
@@ -24,7 +24,7 @@ typedef struct parameters_ {
long num_dsets; /* Number of datasets to create */
long num_elmts; /* Number of native ints in each dset */
int num_iters; /* Number of times to loop doing the IO */
- long buf_size; /* Buffer size */
+ size_t buf_size; /* Buffer size */
} parameters;
typedef struct results_ {