summaryrefslogtreecommitdiffstats
path: root/test/big.c
diff options
context:
space:
mode:
Diffstat (limited to 'test/big.c')
-rw-r--r--test/big.c408
1 files changed, 277 insertions, 131 deletions
diff --git a/test/big.c b/test/big.c
index 8d65559..24b2d25 100644
--- a/test/big.c
+++ b/test/big.c
@@ -16,16 +16,45 @@
/*
* Programmer: Robb Matzke <matzke@llnl.gov>
* Wednesday, April 8, 1998
+ * Modified: Albert Cheng <acheng@hdfgroup.org>
+ * September 11, 2010
+ */
+/*
+ * The purpose of this test is to verify if a virtual file driver can handle:
+ * a. Large file (2GB)
+ * This should exceed 32bits I/O system since offset is a signed
+ * integral type (in order to support negative offset with respect to
+ * end of file).
+ * b. Extra Large file (4GB)
+ * This definite exceeds 32bit I/O and file systems.
+ * c. Huge file (tens of GB)
+ * This verifies the HDF5 library handles big logical file size
+ * correctly.
+ * In practice, if a VFD can handle a big file size, there is no need to
+ * test the smaller file sizes. E.g., If it can handle the Huge file,
+ * there is no need to test the Extra large or Large files. Therefore the
+ * test starts with larger size files and continues to test the smaller size
+ * files only if the large sige file tests have failed.
+ *
+ * Another consideration is that even if a VFD is capable to handle a
+ * huge file but it is likely to take a long time to write every byte
+ * of a huge file. E.g., a simple workstation may have disks of write
+ * speed of 10MB/sec. A huge file of 30GB will take about an hour to
+ * write it. Therefore, this test will run the huge file test only if the
+ * underlying file system supports sparse file. (A Sparse file here means
+ * that disk space is allocated only when the contents are actually written.
+ * E.g., If one creates a new file, seeks forward 10 million bytes, writes
+ * 1 bytes and closes the file, then a sparse file, will show file size of
+ * 10 million bytes but actaully uses only couple disk blocks, much smaller
+ * than the formal file size.)
+ *
+ * One more consideration is that we want to distinguish an HDF5 library
+ * failure from some system limits such as current free disk space or user
+ * disk space quota. Therefore, the test will first attempt to verify no
+ * such limits exist before running the actual VFD tests.
*/
#include "h5test.h"
-const char *FILENAME[] = {
- "big",
- "sec2",
- "stdio",
- NULL
-};
-
#define DNAME "big.data"
#define WRT_N 50
@@ -42,8 +71,29 @@ const char *FILENAME[] = {
# define GB8LL 0 /*cannot do the test*/
#endif
+/* Define Small, Large, Extra Large, Huge File which
+ * corrspond to less than 2GB, 2GB, 4GB, and tens of GB file size.
+ * NOFILE stands for "no file" to be tested.
+ */
+typedef enum fsizes_t { SFILE, LFILE, XLFILE, HUGEFILE, NOFILE} fsizes_t;
+/* Lists of vfd to test */
+typedef enum vfd_t { SEC2_VFD, STDIO_VFD, FAMILY_VFD } vfd_t;
+fsizes_t file_size= NOFILE;
+
+const char *FILENAME[] = {
+ "big",
+ "sec2",
+ "stdio",
+ NULL
+};
+int cflag=1; /* check file system before test */
+int sparse_support=0; /* sparse file supported, default false */
+int have_space=0; /* enough space for huge file test, default false */
+hsize_t family_size_def=FAMILY_SIZE; /* default family file size */
+
/* Protocols */
static void usage(void);
+int testvfd(vfd_t vfd);
/* Array used to record all addresses at which data has been written */
/* so far. Used to prevent overlapping writes. */
@@ -129,7 +179,7 @@ is_sparse(void)
if (5!=HDwrite(fd, "hello", (size_t)5)) return 0;
if (HDclose(fd) < 0) return 0;
if (HDstat("x.h5", &sb) < 0) return 0;
- if (HDunlink("x.h5") < 0) return 0;
+ if (HDremove("x.h5") < 0) return 0;
#ifdef H5_HAVE_STAT_ST_BLOCKS
return ((unsigned long)sb.st_blocks*512 < (unsigned long)sb.st_size);
#else
@@ -156,25 +206,67 @@ is_sparse(void)
*
*-------------------------------------------------------------------------
*/
-static int
-supports_big(void)
+static fsizes_t
+supports_big(vfd_t vfd)
{
- int fd;
+ int fd = -1;
+ fsizes_t fsize = NOFILE;
+
+ switch (vfd){
+ case FAMILY_VFD:
+ case SEC2_VFD:
+ case STDIO_VFD:
+ if ((fd=HDopen("y.h5", O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0)
+ goto error;
- if ((fd=HDopen("y.h5", O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0) return 0;
+ /* Write a few byte at the beginning */
+ if (5!=HDwrite(fd, "hello", (size_t)5))
+ goto quit;
+ fsize = SFILE;
- /* Write a few bytes at 2GB */
- if (HDlseek(fd, 2*GB, SEEK_SET)!=2*GB) return 0;
- if (5!=HDwrite(fd, "hello", (size_t)5)) return 0;
+ /* Write a few bytes at 2GB */
+ if (HDlseek(fd, 2*GB, SEEK_SET)!=2*GB)
+ goto quit;
+ if (5!=HDwrite(fd, "hello", (size_t)5))
+ goto quit;
+ fsize = LFILE;
- /* Write a few bytes at 4GB */
- if (HDlseek(fd, 4*GB, SEEK_SET) != 4*GB) return 0;
- if (5!=HDwrite(fd, "hello", (size_t)5)) return 0;
+ /* Write a few bytes at 4GB */
+ if (HDlseek(fd, 4*GB, SEEK_SET) != 4*GB)
+ goto quit;
+ if (5!=HDwrite(fd, "hello", (size_t)5))
+ goto quit;
+ fsize = XLFILE;
- if (HDclose(fd) < 0) return 0;
- if (HDremove("y.h5") < 0) return 0;
+ /* If this supports sparse_file, write a few bytes at 32GB */
+ if (!sparse_support)
+ goto quit;
+ if (HDlseek(fd, 32*GB, SEEK_SET) != 32*GB)
+ goto quit;
+ if (5!=HDwrite(fd, "hello", (size_t)5))
+ goto quit;
+ fsize = HUGEFILE;
- return (1);
+ break;
+ default:
+ /* unknown or unsupported VFD */
+ goto error;
+ break;
+ }
+
+quit:
+ if (HDclose(fd) < 0)
+ goto error;
+ if (HDremove("y.h5") < 0)
+ goto error;
+ return(fsize);
+
+error:
+ if (fd >= 0){
+ HDclose(fd);
+ HDremove("y.h5");
+ }
+ return (fsize);
}
@@ -231,7 +323,7 @@ enough_room(hid_t fapl)
HDsnprintf(name, sizeof name, filename, i);
if(HDclose(fd[i]) < 0)
ret_value=0;
- HDunlink(name);
+ HDremove(name);
}
return ret_value;
@@ -257,7 +349,7 @@ enough_room(hid_t fapl)
*-------------------------------------------------------------------------
*/
static int
-writer (char* filename, hid_t fapl, int wrt_n)
+writer (char* filename, hid_t fapl, fsizes_t testsize, int wrt_n)
{
hsize_t size1[4] = {8, 1024, 1024, 1024};
hsize_t size2[1] = {GB8LL};
@@ -269,7 +361,39 @@ writer (char* filename, hid_t fapl, int wrt_n)
FILE *out = fopen(DNAME, "w");
hid_t dcpl;
- TESTING("large dataset write");
+ switch(testsize){
+ case LFILE:
+ TESTING("Large dataset write(2GB)");
+ /* reduce size1 to produce a 2GB dataset */
+ size1[1] = 1024/16;
+ size2[0] /= 16;
+ break;
+
+ case XLFILE:
+ TESTING("Extra large dataset write(4GB)");
+ /* reduce size1 to produce a 4GB dataset */
+ size1[1] = 1024/8;
+ size2[0] /= 8;
+ break;
+
+ case HUGEFILE:
+ TESTING("Huge dataset write");
+ /* Leave size1 as 32GB */
+ break;
+
+ case SFILE:
+ TESTING("small dataset write(1GB)");
+ /* reduce size1 to produce a 1GB dataset */
+ size1[1] = 1024/32;
+ size2[0] /= 32;
+ break;
+
+ case NOFILE:
+ /* what to do?? */
+ HDfprintf(stdout, "Unexpected file size of NOFILE\n");
+ goto error;
+ break;
+ }
/*
* We might be on a machine that has 32-bit files, so create an HDF5 file
@@ -475,6 +599,124 @@ usage(void)
+/* Flush stdout at the end of this test routine to ensure later output to */
+/* stderr will not come out before it.*/
+int testvfd(vfd_t vfd)
+{
+ hid_t fapl=-1;
+ hsize_t family_size;
+ char filename[1024];
+ fsizes_t testsize;
+
+
+ switch(vfd){
+ case FAMILY_VFD:
+ /* Test huge file with the family driver */
+ puts("Testing big file with the Family Driver ");
+ if ((fapl=H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ goto error;
+
+ if (H5Pset_fapl_family(fapl, family_size_def, H5P_DEFAULT) < 0)
+ goto error;
+
+ if (cflag){
+ /*
+ * We shouldn't run this test if the file system doesn't support holes
+ * because we would generate multi-gigabyte files.
+ */
+ puts("Checking if file system is adequate for this test...");
+ if (sizeof(long long)<8 || 0==GB8LL) {
+ puts("Test skipped because sizeof(long long) is too small. This");
+ puts("hardware apparently doesn't support 64-bit integer types.");
+ usage();
+ goto quit;
+ }
+ if (!sparse_support) {
+ puts("Test skipped because file system does not support holes.");
+ usage();
+ goto quit;
+ }
+ if (!enough_room(fapl)) {
+ puts("Test skipped because of quota (file size or num open files).");
+ usage();
+ goto quit;
+ }
+ }
+
+ /* Do the test with the Family Driver */
+ h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
+
+ if (writer(filename, fapl, HUGEFILE, WRT_N)) goto error;
+ if (reader(filename, fapl)) goto error;
+
+ puts("Test passed with the Family Driver.");
+ break;
+
+ case SEC2_VFD:
+ testsize = supports_big(SEC2_VFD);
+ if (testsize == NOFILE) {
+ HDfprintf(stdout, "Test for sec2 is skipped because file system does not support big files.\n");
+ goto quit;
+ }
+ /* Test big file with the SEC2 driver */
+ puts("Testing big file with the SEC2 Driver ");
+
+ if ((fapl=H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ goto error;
+ if(H5Pset_fapl_sec2(fapl) < 0)
+ goto error;
+
+ h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
+
+ if (writer(filename, fapl, testsize, WRT_N)) goto error;
+ if (reader(filename, fapl)) goto error;
+
+ puts("Test passed with the SEC2 Driver.");
+ break;
+
+ case STDIO_VFD:
+ testsize = supports_big(STDIO_VFD);
+ if (testsize == NOFILE) {
+ HDfprintf(stdout, "Test for stdio is skipped because file system does not support big files.\n");
+ goto quit;
+ }
+ puts("\nTesting big file with the STDIO Driver ");
+
+ if ((fapl=H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ goto error;
+ if(H5Pset_fapl_stdio(fapl) < 0)
+ goto error;
+
+ h5_fixname(FILENAME[2], fapl, filename, sizeof filename);
+
+ if (writer(filename, fapl, testsize, WRT_N)) goto error;
+ if (reader(filename, fapl)) goto error;
+ puts("Test passed with the STDIO Driver.");
+ break;
+
+ default:
+ puts("Unsupprted VFD");
+ usage();
+ goto error;;
+ } /* end of switch (vfd) */
+
+quit:
+ /* End with normal return code */
+ /* Clean up the test file */
+ if (h5_cleanup(FILENAME, fapl)) HDremove(DNAME);
+ fflush(stdout);
+ return 0;
+
+
+error:
+ if (fapl>=0) H5Pclose(fapl);
+ puts("*** TEST FAILED ***");
+ fflush(stdout);
+ return 1;
+}
+
+
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -501,15 +743,9 @@ usage(void)
int
main (int ac, char **av)
{
- hid_t fapl=-1;
- hsize_t family_size;
- hsize_t family_size_def; /* default family file size */
unsigned long seed = 0; /* Random # seed */
- int cflag=1; /* check file system before test */
- char filename[1024];
/* parameters setup */
- family_size_def = FAMILY_SIZE;
while (--ac > 0){
av++;
@@ -538,6 +774,11 @@ main (int ac, char **av)
}
}
+ /* check sparse file support unless cflag is not set. */
+ if (cflag)
+ sparse_support = is_sparse();
+
+
/* Choose random # seed */
seed = (unsigned long)HDtime(NULL);
#ifdef QAK
@@ -546,113 +787,18 @@ HDfprintf(stderr, "Random # seed was: %lu\n", seed);
#endif /* QAK */
HDsrandom(seed);
- /* Reset library */
- h5_reset();
- fapl = h5_fileaccess();
-
- /* Test big file with the family driver */
- puts("Testing big file with the Family Driver ");
- if (H5FD_FAMILY!=H5Pget_driver(fapl)) {
- HDfprintf(stdout,
- "Changing file drivers to the family driver, %Hu bytes each\n",
- family_size_def);
- if (H5Pset_fapl_family(fapl, family_size_def, H5P_DEFAULT) < 0) goto error;
- } else if (H5Pget_fapl_family(fapl, &family_size, NULL) < 0) {
+/*=================================================*/
+ if (testvfd(FAMILY_VFD) != 0)
+ goto error;
+ if (testvfd(SEC2_VFD) != 0)
+ goto error;
+ if (testvfd(STDIO_VFD) != 0)
goto error;
- } else if (family_size!=family_size_def) {
- HDfprintf(stdout, "Changing family member size from %Hu to %Hu\n",
- family_size, family_size_def);
- if (H5Pset_fapl_family(fapl, family_size_def, H5P_DEFAULT) < 0)
- goto error;
- }
-
- if (cflag){
- /*
- * We shouldn't run this test if the file system doesn't support holes
- * because we would generate multi-gigabyte files.
- */
- puts("Checking if file system is adequate for this test...");
- if (sizeof(long long)<8 || 0==GB8LL) {
- puts("Test skipped because sizeof(long long) is too small. This");
- puts("hardware apparently doesn't support 64-bit integer types.");
- usage();
- goto quit;
- }
- if (!is_sparse()) {
- puts("Test skipped because file system does not support holes.");
- usage();
- goto quit;
- }
- if (!enough_room(fapl)) {
- puts("Test skipped because of quota (file size or num open files).");
- usage();
- goto quit;
- }
- }
-
- /* Do the test with the Family Driver */
- h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
-
- if (writer(filename, fapl, WRT_N)) goto error;
- if (reader(filename, fapl)) goto error;
-
- puts("Test passed with the Family Driver.");
-
- /*
- * We shouldn't run this test if the file system doesn't support big files
- * because we would generate multi-gigabyte files.
- */
- puts("\nChecking if file system supports big files...");
- if (!supports_big()) {
- puts("Tests for sec2 and stdio are skipped because file system does not support big files.");
- usage();
- goto quit;
- }
-
- /* Clean up the test file */
- if (h5_cleanup(FILENAME, fapl)) remove(DNAME);
-
- /* Test big file with the SEC2 driver */
- puts("Testing big file with the SEC2 Driver ");
-
- fapl = h5_fileaccess();
- if(H5Pset_fapl_sec2(fapl) < 0)
-
- HDmemset(filename, 0, sizeof(filename));
- h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
-
- if (writer(filename, fapl, WRT_N)) goto error;
- if (reader(filename, fapl)) goto error;
-
- puts("Test passed with the SEC2 Driver.");
-
-#ifdef H5_HAVE_FSEEKO
- /* Clean up the test file */
- if (h5_cleanup(FILENAME, fapl)) remove(DNAME);
-
- /* Test big file with the STDIO driver only if fseeko is supported,
- * because the OFFSET parameter of fseek has the type LONG, not big
- * enough to support big files. */
- puts("\nTesting big file with the STDIO Driver ");
-
- fapl = h5_fileaccess();
- if(H5Pset_fapl_stdio(fapl) < 0)
-
- HDmemset(filename, 0, sizeof(filename));
- h5_fixname(FILENAME[2], fapl, filename, sizeof filename);
-
- if (writer(filename, fapl, WRT_N)) goto error;
- if (reader(filename, fapl)) goto error;
- puts("Test passed with the STDIO Driver.");
-#endif
-quit:
/* End with normal exit code */
- if (h5_cleanup(FILENAME, fapl)) remove(DNAME);
return 0;
error:
- if (fapl>=0) H5Pclose(fapl);
puts("*** TEST FAILED ***");
return 1;
}