From 071be8ef3b668af0a4b3171cf02cbbdeffd52084 Mon Sep 17 00:00:00 2001 From: Raymond Lu Date: Wed, 22 Jun 2005 13:59:48 -0500 Subject: [svn-r10971] Purpose: Bug fix Description: See details from Bug #213. Family member file size wasn't saved anywhere in file. When family file is opened, the first member size determine the member size. Solution: Hopefully, this is the last step of checkin for this stage. This step mainly enables h5repart tool to be able to convert file driver from family to sec2. Because the library saves family information in file, in trying to convert to sec2 driver, the library simply change the address of driver information to undefined so that the driver information block can be ignored. This step also updates the reference manual of H5Pset_fapl_family() and h5repart. In the fifth step of checkin, backward compatibility with v1.6 is tested. A family file created was created with the v1.6 library and opened with this version of the library. In the fourth step of checkin, a test suit is added for h5repart, including a program to generate the test files, a script file to run h5repart, and a program to verify repartitioned files can be opened by the library. There's a change from the first step of checkin. Family name template is no longer saved in the superblock because different pathname can make the name different. In the third step of checkin, h5repart has been modified. If h5repart is used to change the size of family member file, the new size(actual member size) is saved in the superblock. In the second step of checkin, multi driver is checked against the driver name saved in superblock. Wrong driver will result in a failure with an error message indicating multi driver should be used. This change includes split driver because it's a special case for multi driver. In the first step of checkin. Family member size and name template(unused at this stage) are saved in file superblock. When file is reopened,the size passed in thrin superblock. A different size will trigger a failure with an error message indicating the right size. Wrong driver to open family file will cause a failure, too. Platforms tested: h5committest and fuss. Misc. update: doc/html/RM_H5P.html and doc/html/Tools.html --- doc/html/RM_H5P.html | 10 ++++-- doc/html/Tools.html | 19 ++++++++--- src/H5F.c | 31 ++++++++++++++++-- src/H5Fpkg.h | 3 +- src/H5Fprivate.h | 6 ++++ tools/misc/h5repart.c | 76 ++++++++++++++++++++++++++++--------------- tools/misc/repart_test.c | 43 ++++++++++++++++++++++-- tools/misc/testh5repart.sh.in | 4 ++- 8 files changed, 152 insertions(+), 40 deletions(-) diff --git a/doc/html/RM_H5P.html b/doc/html/RM_H5P.html index 2d0572a..bc6eae6 100644 --- a/doc/html/RM_H5P.html +++ b/doc/html/RM_H5P.html @@ -6799,8 +6799,14 @@ END SUBROUTINE h5pset_fapl_core_f
H5Pset_fapl_family sets the file access property list identifier, fapl_id, to use the family driver.

- memb_size is the size in bytes of each file member - and is used only when creating a new file. + memb_size is the size in bytes of each file member. This size + will be saved in the file when the property list fapl_id is used + to create a new file. If fapl_id is used to open an existing + file, memb_size has to be equal to the original size saved in + the file. A failure with an errror message indicating the correct member + size will be returned if memb_size does not match the size saved. + If any user does not know the original size, H5F_FAMILY_DEFAULT + can be passed in. The library will retrieve the correct size saved in the file.

memb_fapl_id is the identifier of the file access property list to be used for each family member. diff --git a/doc/html/Tools.html b/doc/html/Tools.html index 70f7735..afe27dd 100644 --- a/doc/html/Tools.html +++ b/doc/html/Tools.html @@ -893,16 +893,22 @@ installed with HDF5. [-v] [-V] [-[b|m]N[g|m|k]] + [-family_to_sec2] source_file dest_file

Purpose:
Repartitions a file or family of files.
Description: -
h5repart splits a single file into a family of - files, joins a family of files into a single file, or copies - one family of files to another while changing the size of the - family members. h5repart can also be used to - copy a single file to a single file with holes. +
h5repart joins a family of files into a single file, + or copies one family of files to another while changing the size + of the family members. h5repart can also be used to + copy a single file to a single file with holes. At this stage, + h5repart can not split a single non-family file into + a family of file(s). +

+ To convert a family of file(s) to a single non-family file + (sec2 file), the option -family_to_sec2 + has to be used.

Sizes associated with the -b and -m options may be suffixed with g for gigabytes, @@ -926,6 +932,9 @@ installed with HDF5. -mN The destination member size or 1GB + -family_to_sec2 + Convert file driver from family to sec2 + source_file     The name of the source file diff --git a/src/H5F.c b/src/H5F.c index 54dee30..63e108b 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -213,6 +213,7 @@ H5F_init_interface(void) H5F_close_degree_t close_degree = H5F_CLOSE_DEGREE_DEF; hsize_t family_offset = H5F_ACS_FAMILY_OFFSET_DEF; hsize_t family_newsize = H5F_ACS_FAMILY_NEWSIZE_DEF; + hbool_t family_to_sec2 = H5F_ACS_FAMILY_TO_SEC2_DEF; H5FD_mem_t mem_type = H5F_ACS_MULTI_TYPE_DEF; /* File mount property class variable. @@ -366,7 +367,11 @@ H5F_init_interface(void) /* Register the private property of new family file size. It's used by h5repart only. */ if(H5P_register(acs_pclass,H5F_ACS_FAMILY_NEWSIZE_NAME,H5F_ACS_FAMILY_NEWSIZE_SIZE, &family_newsize,NULL,NULL,NULL,NULL,NULL,NULL,NULL)<0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - + + /* Register the private property of whether convert family to sec2 driver. It's used by h5repart only. */ + if(H5P_register(acs_pclass,H5F_ACS_FAMILY_TO_SEC2_NAME,H5F_ACS_FAMILY_TO_SEC2_SIZE, &family_to_sec2,NULL,NULL,NULL,NULL,NULL,NULL,NULL)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the data type of multi driver info */ if(H5P_register(acs_pclass,H5F_ACS_MULTI_TYPE_NAME,H5F_ACS_MULTI_TYPE_SIZE, &mem_type,NULL,NULL,NULL,NULL,NULL,NULL,NULL)<0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") @@ -1789,6 +1794,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t d hbool_t driver_has_cmp; /*`cmp' callback defined? */ H5P_genplist_t *a_plist; /*file access property list */ H5F_close_degree_t fc_degree; /*file close degree */ + hbool_t fam_sec2=FALSE; /*change family to sec2 driver? */ H5F_t *ret_value; /*actual return value */ FUNC_ENTER_NOAPI(H5F_open, NULL) @@ -1907,6 +1913,20 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t d file->intent = flags; file->name = H5MM_xstrdup(name); + if(NULL == (a_plist = H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + + /* This step is for h5repart tool only. If user wants to change file driver from + * family to sec2 while using h5repart, this private property should be set so that + * in the later step, the library can ignore the family driver information saved + * in the superblock. + */ + if(H5P_exist_plist(a_plist, H5F_ACS_FAMILY_TO_SEC2_NAME) > 0) + if(H5P_get(a_plist, H5F_ACS_FAMILY_TO_SEC2_NAME, &fam_sec2) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get property of changing family to sec2") + + file->shared->fam_to_sec2 = fam_sec2; + /* * Read or write the file superblock, depending on whether the file is * empty or not. @@ -2047,8 +2067,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t d * second time or later, verify the access property list value matches * the degree in shared file structure. */ - if(NULL == (a_plist = H5I_object(fapl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") if(H5P_get(a_plist, H5F_CLOSE_DEGREE_NAME, &fc_degree) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file close degree") @@ -2583,6 +2601,13 @@ H5F_read_superblock(H5F_t *f, hid_t dxpl_id, H5G_entry_t *root_ent, haddr_t addr /* Set the super block checksum */ shared->super_chksum = chksum; + /* This step is for h5repart tool only. If user wants to change file driver from + * family to sec2 while using h5repart, set the driver address to undefined to let + * the library ignore the family driver information saved in the superblock. + */ + if(shared->fam_to_sec2) + shared->driver_addr = HADDR_UNDEF; + /* Decode the optional driver information block */ if (H5F_addr_defined(shared->driver_addr)) { haddr_t drv_addr = shared->base_addr + shared->driver_addr; diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 17dcfdc..c7b9e95 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -92,7 +92,8 @@ typedef struct H5F_file_t { haddr_t base_addr; /* Absolute base address for rel.addrs. */ haddr_t freespace_addr; /* Relative address of free-space info */ haddr_t driver_addr; /* File driver information block address*/ - + hbool_t fam_to_sec2; /* Is h5repart changing driver from family to sec2 */ + unsigned super_chksum; /* Superblock checksum */ unsigned drvr_chksum; /* Driver info block checksum */ H5AC_t *cache; /* The object cache */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index b5f9647..0e66d6d 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -398,6 +398,12 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_FAMILY_NEWSIZE_SIZE sizeof(hsize_t) #define H5F_ACS_FAMILY_NEWSIZE_DEF 0 +/* Definition for whether to conver family to sec2 driver. It's private + * property only used by h5repart */ +#define H5F_ACS_FAMILY_TO_SEC2_NAME "family_to_sec2" +#define H5F_ACS_FAMILY_TO_SEC2_SIZE sizeof(hbool_t) +#define H5F_ACS_FAMILY_TO_SEC2_DEF FALSE + /* Definition for data type in multi file driver */ #define H5F_ACS_MULTI_TYPE_NAME "multi_type" #define H5F_ACS_MULTI_TYPE_SIZE sizeof(H5FD_mem_t) diff --git a/tools/misc/h5repart.c b/tools/misc/h5repart.c index b3a463b..c88b5a8 100644 --- a/tools/misc/h5repart.c +++ b/tools/misc/h5repart.c @@ -65,9 +65,11 @@ # define MIN3(X,Y,Z) MIN(MIN(X,Y),Z) #endif -/*Make this private property(defined in H5Fprivate.h) available to h5repart, - *to update the member file size in the superblock.*/ +/*Make these 2 private properties(defined in H5Fprivate.h) available to h5repart. + *The first one updates the member file size in the superblock. The second one + *change file driver from family to sec2. */ #define H5F_ACS_FAMILY_NEWSIZE_NAME "family_newsize" +#define H5F_ACS_FAMILY_TO_SEC2_NAME "family_to_sec2" /*------------------------------------------------------------------------- @@ -87,12 +89,13 @@ static void usage (const char *progname) { - fprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] SRC DST\n", + fprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] [-family_to_sec2] SRC DST\n", progname); fprintf(stderr, " -v Produce verbose output\n"); fprintf(stderr, " -V Print a version number and exit\n"); fprintf(stderr, " -b N The I/O block size, defaults to 1kB\n"); fprintf(stderr, " -m N The destination member size or 1GB\n"); + fprintf(stderr, " -family_to_sec2 Change file driver from family to sec2\n"); fprintf(stderr, " SRC The name of the source file\n"); fprintf(stderr, " DST The name of the destination files\n"); fprintf(stderr, "Sizes may be suffixed with `g' for GB, `m' for MB or " @@ -230,6 +233,7 @@ main (int argc, char *argv[]) hid_t fapl; /*file access property list */ hid_t file; hsize_t hdsize; /*destination logical memb size */ + hbool_t family_to_sec2=FALSE; /*change family to sec2 driver? */ /* * Get the program name from argv[0]. Use only the last component. @@ -248,6 +252,9 @@ main (int argc, char *argv[]) printf("This is %s version %u.%u release %u\n", prog_name, H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE); exit(0); + } else if (!strcmp (argv[argno], "-family_to_sec2")) { + family_to_sec2 = TRUE; + argno++; } else if ('b'==argv[argno][1]) { blk_size = get_size (prog_name, &argno, argc, argv); } else if ('m'==argv[argno][1]) { @@ -449,36 +456,53 @@ main (int argc, char *argv[]) } close (dst); - /* Modify family size saved in superblock through private property. It signals - * library to save the new member size(specified in command line) in superblock. - * This private property is for this tool only. */ + /* Modify family driver information saved in superblock through private property. + * These private properties are for this tool only. */ if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) { perror ("H5Pcreate"); exit (1); } - if(H5Pset_fapl_family(fapl, H5F_FAMILY_DEFAULT, H5P_DEFAULT) < 0) { - perror ("H5Pset_fapl_family"); - exit (1); - } - - /* Set the property as hsize_t */ - hdsize = dst_size; - if(H5Pset(fapl, H5F_ACS_FAMILY_NEWSIZE_NAME, &hdsize) < 0) { - perror ("H5Pset_family_newsize"); - exit (1); + if(family_to_sec2) { + /* The user wants to change file driver from family to sec2. Open the file + * with sec2 driver. This property signals the library to ignore the family + * driver information saved in the superblock. */ + if(H5Pset(fapl, H5F_ACS_FAMILY_TO_SEC2_NAME, &family_to_sec2) < 0) { + perror ("H5Pset"); + exit (1); + } + } else { + /* Modify family size saved in superblock through private property. It signals + * library to save the new member size(specified in command line) in superblock. + * This private property is for this tool only. */ + if(H5Pset_fapl_family(fapl, H5F_FAMILY_DEFAULT, H5P_DEFAULT) < 0) { + perror ("H5Pset_fapl_family"); + exit (1); + } + + /* Set the property of the new member size as hsize_t */ + hdsize = dst_size; + if(H5Pset(fapl, H5F_ACS_FAMILY_NEWSIZE_NAME, &hdsize) < 0) { + perror ("H5Pset"); + exit (1); + } } - /* Open file for "read and write" to flush metadata. Flushing metadata - * will update the superblock to the new member size. */ - if((file=H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl))<0) { - perror ("H5Fopen"); - exit (1); - } - - if(H5Fclose(file)<0) { - perror ("H5Fclose"); - exit (1); + /* If the new file is a family file, try to open file for "read and write" to + * flush metadata. Flushing metadata will update the superblock to the new + * member size. If the original file is a family file and the new file is a sec2 + * file, the property FAMILY_TO_SEC2 will signal the library to switch to sec2 + * driver when the new file is opened. If the original file is a sec2 file and the + * new file can only be a sec2 file, reopen the new file should fail. There's + * nothing to do in this case. */ + H5E_BEGIN_TRY { + file=H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl); + } H5E_END_TRY; + if(file>=0) { + if(H5Fclose(file)<0) { + perror ("H5Fclose"); + exit (1); + } } if(H5Pclose(fapl)<0) { diff --git a/tools/misc/repart_test.c b/tools/misc/repart_test.c index 7b81cf4..833a496 100644 --- a/tools/misc/repart_test.c +++ b/tools/misc/repart_test.c @@ -30,11 +30,12 @@ const char *FILENAME[] = { "fst_family%05d.h5", "scd_family%05d.h5", + "family_to_sec2.h5", NULL }; -herr_t -test_family_h5repart_opens(void); +herr_t test_family_h5repart_opens(void); +herr_t test_sec2_h5repart_opens(void); /*------------------------------------------------------------------------- @@ -91,6 +92,43 @@ error: /*------------------------------------------------------------------------- + * Function: test_sec2_h5repart_opens + * + * Purpose: Tries to reopen a sec2 file. + * + * Return: Success: exit(0) + * + * Failure: exit(1) + * + * Programmer: Raymond Lu + * June 21, 2005 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +test_sec2_h5repart_opens(void) +{ + hid_t file=(-1); + + /* open the sec2 file */ + if((file=H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT))<0) + goto error; + + if(H5Fclose(file)<0) + goto error; + + return 0; + +error: + H5E_BEGIN_TRY { + H5Fclose(file); + } H5E_END_TRY; + return -1; +} + + +/*------------------------------------------------------------------------- * Function: main * * Purpose: Tests h5repart-ed family files @@ -112,6 +150,7 @@ main(void) int nerrors=0; nerrors += test_family_h5repart_opens()<0 ?1:0; + nerrors += test_sec2_h5repart_opens()<0 ?1:0; if (nerrors) goto error; diff --git a/tools/misc/testh5repart.sh.in b/tools/misc/testh5repart.sh.in index 2fb90ea..744dd00 100644 --- a/tools/misc/testh5repart.sh.in +++ b/tools/misc/testh5repart.sh.in @@ -92,6 +92,8 @@ SKIP() { TOOLTEST -m 20000 family_file%05d.h5 $actual_dir/fst_family%05d.h5 # repartition family member size to 5 KB. TOOLTEST -m 5k family_file%05d.h5 $actual_dir/scd_family%05d.h5 +# convert family file to sec2 file of 20,000 bytes +TOOLTEST -m 20000 -family_to_sec2 family_file%05d.h5 $actual_dir/family_to_sec2.h5 # test the output files repartitioned above. OUTPUTTEST @@ -104,7 +106,7 @@ fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then cd $actual_dir - rm -f fst_family*.h5 scd_family*.h5 + rm -f fst_family*.h5 scd_family*.h5 family_to_sec2.h5 fi exit $nerrors -- cgit v0.12