/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * * Copyright by the Board of Trustees of the University of Illinois. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * * the files COPYING and Copyright.html. COPYING can be found at the root * * of the source code distribution tree; Copyright.html can be found at the * * root level of an installed copy of the electronic HDF5 document set and * * is linked from the top-level documents page. It can also be found at * * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * * access to either file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* * Programmer: Robb Matzke * Wednesday, April 8, 1998 * Modified: Albert Cheng * September 11, 2010 */ /* * The purpose of this test is to verify if a virtual file driver can handle: * a. Large file (2GB) * This should exceed 32bits I/O system since offset is a signed * integral type (in order to support negative offset with respect to * end of file). * b. Extra Large file (4GB) * This definite exceeds 32bit I/O and file systems. * c. Huge file (tens of GB) * This verifies the HDF5 library handles big logical file size * correctly. * In practice, if a VFD can handle a big file size, there is no need to * test the smaller file sizes. E.g., If it can handle the Huge file, * there is no need to test the Extra large or Large files. Therefore the * test starts with larger size files and continues to test the smaller size * files only if the large sige file tests have failed. * * Another consideration is that even if a VFD is capable to handle a * huge file but it is likely to take a long time to write every byte * of a huge file. E.g., a simple workstation may have disks of write * speed of 10MB/sec. A huge file of 30GB will take about an hour to * write it. Therefore, this test will run the huge file test only if the * underlying file system supports sparse file. (A Sparse file here means * that disk space is allocated only when the contents are actually written. * E.g., If one creates a new file, seeks forward 10 million bytes, writes * 1 bytes and closes the file, then a sparse file, will show file size of * 10 million bytes but actaully uses only couple disk blocks, much smaller * than the formal file size.) * * One more consideration is that we want to distinguish an HDF5 library * failure from some system limits such as current free disk space or user * disk space quota. Therefore, the test will first attempt to verify no * such limits exist before running the actual VFD tests. */ #include "h5test.h" /* Define Large file, Extra Large file, Huge File */ typedef enum fsizes_t { LFILE, XLFILE, HUGEFILE} fsizes_t; /* Lists of vfd to test */ typedef enum vfd_t { SEC2_VFD, STDIO_VFD, FAMILY_VFD } vfd_t; fsizes_t file_size= HUGEFILE; const char *FILENAME[] = { "big", "sec2", "stdio", NULL }; int cflag=1; /* check file system before test */ int sparse_support=0; /* sparse file supported, default false */ int have_space=0; /* enough space for huge file test, default false */ hsize_t family_size_def; /* default family file size */ #define DNAME "big.data" #define WRT_N 50 #define WRT_SIZE 4*1024 #define FAMILY_SIZE 1024*1024*1024 #define GB (HDoff_t)0x40000000L #define MAX_TRIES 100 #if H5_SIZEOF_LONG_LONG >= 8 # define GB8LL ((unsigned long long)8*1024*1024*1024) #else # define GB8LL 0 /*cannot do the test*/ #endif /* Protocols */ static void usage(void); int testvfd(vfd_t vfd); /* Array used to record all addresses at which data has been written */ /* so far. Used to prevent overlapping writes. */ static hsize_t values_used[WRT_N]; /*------------------------------------------------------------------------- * Function: randll * * Purpose: Create a random long long value. * Ensures that a write at this value doesn't overlap any * previous write. * * Return: Success: Random value * * Failure: Random value which overlaps another write * * Programmer: Robb Matzke * Tuesday, November 24, 1998 * * Modifications: * *------------------------------------------------------------------------- */ static hsize_t randll(hsize_t limit, int current_index) { hsize_t acc; int overlap = 1; int i; int tries = 0; /* Generate up to MAX_TRIES random numbers until one of them */ /* does not overlap with any previous writes */ while(overlap != 0 && tries < MAX_TRIES) { acc = HDrandom(); acc *= HDrandom(); acc = acc % limit; overlap = 0; for(i = 0; i < current_index; i++) { if((acc >= values_used[i]) && (acc < values_used[i]+WRT_SIZE)) overlap = 1; if((acc+WRT_SIZE >= values_used[i]) && (acc+WRT_SIZE < values_used[i]+WRT_SIZE)) overlap = 1; } tries++; } values_used[current_index]=acc; return acc; } /*------------------------------------------------------------------------- * Function: is_sparse * * Purpose: Determines if the file system of the current working * directory supports holes. * * Return: Success: Non-zero if holes are supported; zero * otherwise. * * Failure: zero * * Programmer: Robb Matzke * Wednesday, July 15, 1998 * * Modifications: * *------------------------------------------------------------------------- */ static int is_sparse(void) { int fd; h5_stat_t sb; if ((fd=HDopen("x.h5", O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0) return 0; if (HDlseek(fd, (off_t)(1024*1024), SEEK_SET)!=1024*1024) return 0; if (5!=HDwrite(fd, "hello", (size_t)5)) return 0; if (HDclose(fd) < 0) return 0; if (HDstat("x.h5", &sb) < 0) return 0; if (HDremove("x.h5") < 0) return 0; #ifdef H5_HAVE_STAT_ST_BLOCKS return ((unsigned long)sb.st_blocks*512 < (unsigned long)sb.st_size); #else return (0); #endif } /*------------------------------------------------------------------------- * Function: supports_big * * Purpose: Determines if the file system of the current working * directory supports big files. * * Return: Success: Non-zero if big files are supported; zero * otherwise. * * Failure: zero * * Programmer: Raymond Lu * Wednesday, April 18, 2007 * * Modifications: * *------------------------------------------------------------------------- */ static int supports_big(vfd_t vfd) { int fd; switch (vfd){ case FAMILY_VFD: case SEC2_VFD: case STDIO_VFD: if ((fd=HDopen("y.h5", O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0) return 0; /* Write a few bytes at 2GB */ if (HDlseek(fd, 2*GB, SEEK_SET)!=2*GB) return 0; if (5!=HDwrite(fd, "hello", (size_t)5)) return 0; /* Write a few bytes at 4GB */ if (HDlseek(fd, 4*GB, SEEK_SET) != 4*GB) return 0; if (5!=HDwrite(fd, "hello", (size_t)5)) return 0; if (HDclose(fd) < 0) return 0; if (HDremove("y.h5") < 0) return 0; break; default: /* unknown or unsupported VFD */ return 0; break; } return (1); } /*------------------------------------------------------------------------- * Function: enough_room * * Purpose: Tries to create a bunch of sparse files to see if quotas will * get in the way. Some systems also have problems opening * enough files and we'll check that too. * * Return: Success: Non-zero * * Failure: zero * * Programmer: Robb Matzke * Thursday, August 6, 1998 * * Modifications: * *------------------------------------------------------------------------- */ static int enough_room(hid_t fapl) { int ret_value=0; int fd[68]; size_t i, size = (size_t)1 << 30; char filename[1024], name[1024]; /* Initialize file descriptors */ for (i=0; i=0; i++) { HDsnprintf(name, sizeof name, filename, i); if(HDclose(fd[i]) < 0) ret_value=0; HDremove(name); } return ret_value; } /*------------------------------------------------------------------------- * Function: writer * * Purpose: Creates a *big* dataset. * * Return: Success: 0 * * Failure: >0 * * Programmer: Robb Matzke * Wednesday, April 8, 1998 * * Modifications: * Robb Matzke, 15 Jul 1998 * Addresses are written to the file DNAME instead of stdout. * *------------------------------------------------------------------------- */ static int writer (char* filename, hid_t fapl, int wrt_n) { hsize_t size1[4] = {8, 1024, 1024, 1024}; hsize_t size2[1] = {GB8LL}; hsize_t hs_start[1]; hsize_t hs_size[1]; hid_t file=-1, space1=-1, space2=-1, mem_space=-1, d1=-1, d2=-1; int *buf = (int*)malloc (sizeof(int) * WRT_SIZE); int i, j; FILE *out = fopen(DNAME, "w"); hid_t dcpl; TESTING("large dataset write"); /* * We might be on a machine that has 32-bit files, so create an HDF5 file * which is a family of files. Each member of the family will be 1GB */ if ((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) { goto error; } /* Create simple data spaces according to the size specified above. */ if ((space1 = H5Screate_simple (4, size1, size1)) < 0 || (space2 = H5Screate_simple (1, size2, size2)) < 0) { goto error; } /* Create the datasets */ /* * The fix below is provided for bug#921 * H5Dcreate with H5P_DEFAULT creation properties * will create a set of solid 1GB files; test will crash if quotas are enforced * or it will take some time to write a file. * We should create a dataset allocating space late and never writing fill values. * EIP 4/8/03 */ dcpl = H5Pcreate(H5P_DATASET_CREATE); H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_LATE); H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER); if((d1 = H5Dcreate2(file, "d1", H5T_NATIVE_INT, space1, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0 || (d2 = H5Dcreate2(file, "d2", H5T_NATIVE_INT, space2, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { goto error; } /* Write some things to them randomly */ hs_size[0] = WRT_SIZE; if ((mem_space = H5Screate_simple (1, hs_size, hs_size)) < 0) goto error; for (i=0; i0 * * Programmer: Robb Matzke * Friday, April 10, 1998 * * Modifications: * *------------------------------------------------------------------------- */ static int reader(char *filename, hid_t fapl) { FILE *script = NULL; hid_t file = -1, mspace = -1, fspace = -1, d2 = -1; char ln[128], *s; hsize_t hs_offset[1]; hsize_t hs_size[1] = {WRT_SIZE}; int *buf = (int *)malloc(sizeof(int) * WRT_SIZE); int i, j, zero, wrong, nerrors = 0; /* Open script file */ script = fopen(DNAME, "r"); /* Open HDF5 file */ if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) FAIL_STACK_ERROR /* Open the dataset */ if((d2 = H5Dopen2(file, "d2", H5P_DEFAULT)) < 0) FAIL_STACK_ERROR if((fspace = H5Dget_space(d2)) < 0) FAIL_STACK_ERROR /* Describe `buf' */ if((mspace = H5Screate_simple(1, hs_size, hs_size)) < 0) FAIL_STACK_ERROR /* Read each region */ while(fgets(ln, (int)sizeof(ln), script)) { if('#' != ln[0]) break; i = (int)strtol(ln + 1, &s, 10); hs_offset[0] = HDstrtoll(s, NULL, 0); HDfprintf(stdout, "#%03d 0x%016Hx%47s", i, hs_offset[0], ""); fflush(stdout); if(H5Sselect_hyperslab(fspace, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) FAIL_STACK_ERROR if(H5Dread(d2, H5T_NATIVE_INT, mspace, fspace, H5P_DEFAULT, buf) < 0) FAIL_STACK_ERROR /* Check */ for(j = zero = wrong = 0; j < WRT_SIZE; j++) { if(0 == buf[j]) zero++; else if(buf[j] != i + 1) wrong++; } if(zero) { H5_FAILED(); printf(" %d zero%s\n", zero, 1 == zero ? "" : "s"); } else if(wrong) { SKIPPED(); puts(" Possible overlap with another region."); nerrors++; } else { PASSED(); } } if(H5Dclose(d2) < 0) FAIL_STACK_ERROR if(H5Sclose(mspace) < 0) FAIL_STACK_ERROR if(H5Sclose(fspace) < 0) FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR free(buf); fclose(script); return nerrors; error: H5E_BEGIN_TRY { H5Dclose(d2); H5Sclose(mspace); H5Sclose(fspace); H5Fclose(file); } H5E_END_TRY; if(buf) free(buf); if(script) fclose(script); return 1; } /*------------------------------------------------------------------------- * Function: usage * * Purpose: Print command usage * * Return: void * * Programmer: Albert Chent * Mar 28, 2002 * * Modifications: * *------------------------------------------------------------------------- */ static void usage(void) { HDfprintf(stdout, "Usage: big [-h] [-c] [-fsize }\n" "\t-h\tPrint the help page\n" "\t-c\tFile system Checking skipped. Caution: this test generates\n" "\t\tmany big files and may fill up the file system.\n" "\t-fsize\tChange family size default to where is\n" "\t\ta positive float point number. Default value is %Hu.\n" "Examples:\n" "\tbig -fsize 2.1e9 \t# test with file size just under 2GB\n" "\tbig -fsize 2.2e9 \t# test with file size just above 2GB\n" "\tBe sure the file system can support the file size requested\n" , (hsize_t)FAMILY_SIZE); } /* Flush stdout at the end of this test routine to ensure later output to */ /* stderr will not come out before it.*/ int testvfd(vfd_t vfd) { hid_t fapl=-1; hsize_t family_size; char filename[1024]; switch(vfd){ case FAMILY_VFD: /* Why should I do h5_fileaccess to get fapl and prompty override it??*/ fapl = h5_fileaccess(); /* Test big file with the family driver */ puts("Testing big file with the Family Driver "); if (H5FD_FAMILY!=H5Pget_driver(fapl)) { HDfprintf(stdout, "Changing file drivers to the family driver, %Hu bytes each\n", family_size_def); if (H5Pset_fapl_family(fapl, family_size_def, H5P_DEFAULT) < 0) goto error; } else if (H5Pget_fapl_family(fapl, &family_size, NULL) < 0) { goto error; } else if (family_size!=family_size_def) { HDfprintf(stdout, "Changing family member size from %Hu to %Hu\n", family_size, family_size_def); if (H5Pset_fapl_family(fapl, family_size_def, H5P_DEFAULT) < 0) goto error; } if (cflag){ /* * We shouldn't run this test if the file system doesn't support holes * because we would generate multi-gigabyte files. */ puts("Checking if file system is adequate for this test..."); if (sizeof(long long)<8 || 0==GB8LL) { puts("Test skipped because sizeof(long long) is too small. This"); puts("hardware apparently doesn't support 64-bit integer types."); usage(); goto quit; } if (!is_sparse()) { puts("Test skipped because file system does not support holes."); usage(); goto quit; } if (!enough_room(fapl)) { puts("Test skipped because of quota (file size or num open files)."); usage(); goto quit; } } /* Do the test with the Family Driver */ h5_fixname(FILENAME[0], fapl, filename, sizeof filename); if (writer(filename, fapl, WRT_N)) goto error; if (reader(filename, fapl)) goto error; puts("Test passed with the Family Driver."); break; case SEC2_VFD: /* * We shouldn't run this test if the file system doesn't support big files * because we would generate multi-gigabyte files. */ puts("\nChecking if file system supports big files..."); if (!supports_big(SEC2_VFD)) { puts("Test for sec2 is skipped because file system does not support big files."); usage(); goto quit; } /* Test big file with the SEC2 driver */ puts("Testing big file with the SEC2 Driver "); fapl = h5_fileaccess(); if(H5Pset_fapl_sec2(fapl) < 0) HDmemset(filename, 0, sizeof(filename)); h5_fixname(FILENAME[1], fapl, filename, sizeof filename); if (writer(filename, fapl, WRT_N)) goto error; if (reader(filename, fapl)) goto error; puts("Test passed with the SEC2 Driver."); break; case STDIO_VFD: /* * We shouldn't run this test if the file system doesn't support big files * because we would generate multi-gigabyte files. */ puts("\nChecking if file system supports big files..."); if (!supports_big(STDIO_VFD)) { puts("Test for stdio is skipped because file system does not support big files."); usage(); goto quit; } /* Test big file with the STDIO driver only if fseeko is supported, * because the OFFSET parameter of fseek has the type LONG, not big * enough to support big files. */ puts("\nTesting big file with the STDIO Driver "); fapl = h5_fileaccess(); if(H5Pset_fapl_stdio(fapl) < 0) HDmemset(filename, 0, sizeof(filename)); h5_fixname(FILENAME[2], fapl, filename, sizeof filename); if (writer(filename, fapl, WRT_N)) goto error; if (reader(filename, fapl)) goto error; puts("Test passed with the STDIO Driver."); break; default: puts("Unsupprted VFD"); usage(); goto error;; } /* end of switch (vfd) */ quit: /* End with normal return code */ /* Clean up the test file */ if (h5_cleanup(FILENAME, fapl)) HDremove(DNAME); fflush(stdout); return 0; error: if (fapl>=0) H5Pclose(fapl); puts("*** TEST FAILED ***"); fflush(stdout); return 1; } /*------------------------------------------------------------------------- * Function: main * * Purpose: * * Return: Success: * * Failure: * * Programmer: Robb Matzke * Friday, April 10, 1998 * * Modifications: * Albert Cheng, 2002/03/28 * Added command option -fsize. * Albert Cheng, 2002/04/19 * Added command option -c. * * Raymond Lu, 2007/05/25 * Added similar tests for SEC2 and STDIO drivers. * *------------------------------------------------------------------------- */ int main (int ac, char **av) { unsigned long seed = 0; /* Random # seed */ /* parameters setup */ family_size_def = FAMILY_SIZE; while (--ac > 0){ av++; if (strcmp("-fsize", *av)==0){ /* specify a different family file size */ ac--; av++; if (ac > 0) { family_size_def = (hsize_t)HDstrtoull(*av, NULL, 0); } else{ printf("***Missing fsize value***\n"); usage(); return 1; } } else if (strcmp("-c", *av)==0){ /* turn off file system check before test */ cflag=0; } else if (strcmp("-h", *av)==0){ usage(); return 0; }else{ usage(); return 1; } } /* check sparse file support unless cflag is set. */ if (!cflag) sparse_support = is_sparse(); /* Choose random # seed */ seed = (unsigned long)HDtime(NULL); #ifdef QAK /* seed = (unsigned long)1155438845; */ HDfprintf(stderr, "Random # seed was: %lu\n", seed); #endif /* QAK */ HDsrandom(seed); /*=================================================*/ if (testvfd(FAMILY_VFD) != 0) goto error; if (testvfd(SEC2_VFD) != 0) goto error; if (testvfd(STDIO_VFD) != 0) goto error; /* End with normal exit code */ return 0; error: puts("*** TEST FAILED ***"); return 1; }