From 23cd7c534be3305ef4886866befa3a4fdceab654 Mon Sep 17 00:00:00 2001 From: Pedro Vicente Nunes Date: Tue, 26 Aug 2008 11:02:44 -0500 Subject: [svn-r15533] #1184 Add a userblock to an HDF5 file during the repack. The user gives give a filename and userblock size as command line parameters to h5repack and the contents of that file are stored in the userblock for the HDF5 file created by h5repack. New flags to handle this -u and -b Tested : windows, linux --- tools/h5repack/h5repack.c | 30 ++++++- tools/h5repack/h5repack.h | 16 ++-- tools/h5repack/h5repack.sh.in | 5 ++ tools/h5repack/h5repack_copy.c | 178 +++++++++++++++++++++++++++++++++++++++-- tools/h5repack/h5repack_main.c | 38 ++++++--- tools/h5repack/h5repacktst.c | 89 +++++++++++++++++++-- 6 files changed, 323 insertions(+), 33 deletions(-) diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 954f3e7..73f78f6 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -45,7 +45,7 @@ static int have_request(pack_opt_t *options); * Purpose: locate all high-level HDF5 objects in the file * and compress/chunk them using options * - * Algorythm: 2 traversals are made to the file; the 1st builds a list of + * Algorithm: 2 traversals are made to the file; the 1st builds a list of * the objects, the 2nd makes a copy of them, using the options; * the reason for the 1st traversal is to check for invalid * object name requests @@ -384,8 +384,12 @@ static int check_options(pack_opt_t *options) is present with other objects\n"); return -1; } - - /* check options for the latest format */ + + /*------------------------------------------------------------------------- + * check options for the latest format + *------------------------------------------------------------------------- + */ + if (options->grp_compact < 0) { error_msg(progname, "invalid maximum number of links to store as header messages\n"); return -1; @@ -404,6 +408,26 @@ static int check_options(pack_opt_t *options) return -1; } } + + + /*-------------------------------------------------------------------------------- + * verify new user userblock options; both file name and block size must be present + *--------------------------------------------------------------------------------- + */ + if ( options->ublock_filename != NULL && options->ublock_size == 0 ) + { + error_msg(progname, "user block size missing for file %s\n", + options->ublock_filename); + return -1; + } + + if ( options->ublock_filename == NULL && options->ublock_size != 0 ) + { + error_msg(progname, "file name missing for user block\n", + options->ublock_filename); + return -1; + } + return 0; } diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index baa888e..42b9bbc 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -19,17 +19,12 @@ #include "hdf5.h" #include "h5trav.h" -#include "H5Zprivate.h" /* H5Z_COMMON_CD_VALUES */ - - #define H5FOPENERROR "unable to open file" - -#define PFORMAT "%-7s %-7s %-7s\n" /*chunk info, compression info, name*/ -#define PFORMAT1 "%-7s %-7s %-7s" /*chunk info, compression info, name*/ - -#define MAX_NC_NAME 256 /* max length of a name */ -#define MAX_VAR_DIMS 32 /* max per variable dimensions */ +#define PFORMAT "%-7s %-7s %-7s\n" /* chunk info, compression info, name*/ +#define PFORMAT1 "%-7s %-7s %-7s" /* chunk info, compression info, name*/ +#define MAX_NC_NAME 256 /* max length of a name */ +#define MAX_VAR_DIMS 32 /* max per variable dimensions */ /*------------------------------------------------------------------------- * data structures for command line options @@ -53,7 +48,6 @@ typedef struct { H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression */ -/* #define CD_VALUES H5Z_COMMON_CD_VALUES */ #define CD_VALUES 20 typedef struct { @@ -112,6 +106,8 @@ typedef struct { int grp_indexed; /* Set the minimum number of links to store in the indexed format */ int msg_size[8]; /* Minumum size of shared messages: dataspace, datatype, fill value, filter pipleline, attribute */ + const char *ublock_filename; /* user block file name */ + int ublock_size; /* user block size */ } pack_opt_t; diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index e6d9a98..28809eb 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -490,6 +490,11 @@ else TOOLTEST0 $arg fi +# add a userblock to file +arg="$FILE1 -u ublock.bin -b 2048" +TOOLTEST $arg + + if test $nerrors -eq 0 ; then echo "All $H5REPACK tests passed." diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 73b201e..35bf1c9 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -23,6 +23,10 @@ extern char *progname; +#if 0 +#define H5REPACK_DEBUG +#endif + /*------------------------------------------------------------------------- * macros *------------------------------------------------------------------------- @@ -39,6 +43,7 @@ static void print_dataset_info(hid_t dcpl_id,char *objname,double per, int pr); static int do_copy_objects(hid_t fidin,hid_t fidout,trav_table_t *travt,pack_opt_t *options); static int copy_attr(hid_t loc_in,hid_t loc_out,pack_opt_t *options); static int copy_user_block(const char *infile, const char *outfile, hsize_t size); +static void print_user_block(const char *filename, hid_t fid); /*------------------------------------------------------------------------- * Function: copy_objects @@ -55,10 +60,12 @@ static int copy_user_block(const char *infile, const char *outfile, hsize_t si * Peter Cao, June 13, 2007 * Add "-L, --latest" and other options to pack a file with the latest file format * - * Modification: * Peter Cao, September 25, 2007 * Copy user block when repacking a file * + * Pedro Vicente, August 20, 2008 + * Add a user block to file if requested + * *------------------------------------------------------------------------- */ @@ -84,7 +91,7 @@ int copy_objects(const char* fnamein, /* get user block size */ { - hid_t fcpl_in; /* File creation property list ID for input file */ + hid_t fcpl_in; /* file creation property list ID for input file */ if((fcpl_in = H5Fget_create_plist(fidin)) < 0) { error_msg(progname, "failed to retrieve file creation property list\n"); @@ -182,14 +189,84 @@ int copy_objects(const char* fnamein, } /* end if */ } /* end if */ } /* end if */ + + + + +#if defined (H5REPACK_DEBUG) + print_user_block(fnamein,fidin); +#endif + + + /*------------------------------------------------------------------------- + * set the new user userblock options in the FCPL (before H5Fcreate ) + *------------------------------------------------------------------------- + */ - if((fidout = H5Fcreate(fnameout,H5F_ACC_TRUNC, fcpl, fapl)) < 0) { + if ( options->ublock_size > 0 ) + { + /* either use the FCPL already created or create a new one */ + if(fcpl != H5P_DEFAULT) + { + /* set user block size */ + if(H5Pset_userblock(fcpl, options->ublock_size) < 0) + { + error_msg(progname, "failed to set userblock size\n"); + goto out; + } + + } + + else + { + + /* create a file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + { + error_msg(progname, "fail to create a file creation property list\n"); + goto out; + } + + /* set user block size */ + if(H5Pset_userblock(fcpl, options->ublock_size) < 0) + { + error_msg(progname, "failed to set userblock size\n"); + goto out; + } + + } + + + + } + + /*------------------------------------------------------------------------- + * create the output file + *------------------------------------------------------------------------- + */ + + + if(options->verbose) + printf("Making file <%s>...\n",fnameout); + + + if((fidout = H5Fcreate(fnameout,H5F_ACC_TRUNC, fcpl, fapl)) < 0) + { error_msg(progname, "<%s>: Could not create file\n", fnameout ); goto out; } /* end if */ - if(options->verbose) - printf("Making file <%s>...\n",fnameout); + + /*------------------------------------------------------------------------- + * write a new user block if requested + *------------------------------------------------------------------------- + */ + if ( options->ublock_size > 0 ) + { + copy_user_block( options->ublock_filename, fnameout, options->ublock_size); + } + + /* init table */ trav_table_init(&travt); @@ -235,7 +312,8 @@ int copy_objects(const char* fnamein, H5Fclose(fidin); H5Fclose(fidout); - if(ub_size > 0) + /* write only the input file user block if there is no user block file input */ + if(ub_size > 0 && options->ublock_size == 0) copy_user_block(fnamein, fnameout, ub_size); return 0; @@ -1240,3 +1318,91 @@ done: return status; } + + +/*------------------------------------------------------------------------- + * Function: print_user_block + * + * Purpose: print user block + * + * Return: 0, ok, -1 no + * + * Programmer: Pedro Vicente + * + * Date: August, 20, 2008 + * + *------------------------------------------------------------------------- + */ +static +void print_user_block(const char *filename, hid_t fid) +{ + int fh; /* file handle */ + hsize_t ub_size; /* user block size */ + hsize_t size; /* size read */ + hid_t fcpl; /* file creation property list ID for HDF5 file */ + int i; + + /* get user block size */ + if(( fcpl = H5Fget_create_plist(fid)) < 0) + { + error_msg(progname, "failed to retrieve file creation property list\n"); + goto done; + } + + if(H5Pget_userblock(fcpl, &ub_size) < 0) + { + error_msg(progname, "failed to retrieve userblock size\n"); + goto done; + } + + if(H5Pclose(fcpl) < 0) + { + error_msg(progname, "failed to close property list\n"); + goto done; + } + + /* open file */ + if((fh = HDopen(filename, O_RDONLY, 0)) < 0) + { + goto done; + } + + size = ub_size; + + /* read file */ + while(size > 0) + { + ssize_t nread; /* # of bytes read */ + char rbuf[USERBLOCK_XFER_SIZE]; /* buffer for reading */ + + /* read buffer */ + if(size > USERBLOCK_XFER_SIZE) + nread = HDread(fh, rbuf, (size_t)USERBLOCK_XFER_SIZE); + else + nread = HDread(fh, rbuf, (size_t)size); + + for(i = 0; i < nread; i++) + { + + printf("%c ", rbuf[i]); + + } + printf("\n"); + + if(nread < 0) + { + goto done; + } + + + /* update size of userblock left to transfer */ + size -= nread; + } + +done: + if(fh > 0) + HDclose(fh); + + + return; +} diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 73b6361..c949732 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -37,7 +37,7 @@ static int has_i_o = 0; * Command-line options: The user can specify short or long-named * parameters. */ -static const char *s_opts = "hVvf:l:m:e:nLc:d:s:"; +static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:"; static struct long_options l_opts[] = { { "help", no_arg, 'h' }, { "version", no_arg, 'V' }, @@ -51,6 +51,8 @@ static struct long_options l_opts[] = { { "compact", require_arg, 'c' }, { "indexed", require_arg, 'd' }, { "ssize", require_arg, 's' }, + { "ublock", require_arg, 'u' }, + { "block", require_arg, 'b' }, { NULL, 0, '\0' } }; @@ -84,14 +86,16 @@ static struct long_options l_opts[] = { * added support for multiple global filters * PVN, May 16, 2008 * added backward compatibility for -i infile -o outfile - *------------------------------------------------------------------------- + * PVN, August 20, 2008 + * add a user block to repacked file (switches -u -b) + *------------------------------------------------------------------------- */ int main(int argc, char **argv) { char *infile = NULL; char *outfile = NULL; pack_opt_t options; /*the global options */ - int ret; + int ret=-1; int i; /* initialize options */ @@ -284,7 +288,8 @@ int main(int argc, char **argv) } - + + /* pack it */ ret=h5repack(infile,outfile,&options); @@ -324,6 +329,8 @@ static void usage(const char *prog) printf(" -s S[:F], --ssize=S[:F] Shared object header message minimum size\n"); printf(" -m T, --threshold=T Do not apply the filter to datasets smaller than T\n"); printf(" -e M, --file=M Name of file M with the -f and -l options\n"); + printf(" -u U, --ublock=U Name of file U with user block data to be added\n"); + printf(" -b D, --block=D Size of user block to be added\n"); printf(" -f FILT, --filter=FILT Filter type\n"); printf(" -l LAYT, --layout=LAYT Layout type\n"); @@ -331,7 +338,9 @@ static void usage(const char *prog) printf(" T - is an integer greater than 1, size of dataset in bytes \n"); printf(" M - is a filename.\n"); + printf(" U - is a filename.\n"); printf(" S - is an integer\n"); + printf(" D - is the user block size (any power of 2 equal to 512 or greater)\n"); printf(" F - is the shared object header message type, any of . If F is not specified, S applies to all messages\n"); @@ -510,19 +519,19 @@ static void parse_command_line(int argc, const char* argv[], pack_opt_t* options strcpy(msgType, msgPtr+1); msgPtr[0] = '\0'; ssize = atoi( opt_arg ); - if (strcmp(msgType, "dspace") == 0) { + if (strncmp(msgType, "dspace",6) == 0) { options->msg_size[0] = ssize; } - else if (strcmp(msgType, "dtype") == 0) { + else if (strncmp(msgType, "dtype", 5) == 0) { options->msg_size[1] = ssize; } - else if (strcmp(msgType, "fill") == 0) { + else if (strncmp(msgType, "fill", 4) == 0) { options->msg_size[2] = ssize; } - else if (strcmp(msgType, "pline") == 0) { + else if (strncmp(msgType, "pline", 5) == 0) { options->msg_size[3] = ssize; } - else if (strcmp(msgType, "attr") == 0) { + else if (strncmp(msgType, "attr", 4) == 0) { options->msg_size[4] = ssize; } } @@ -530,6 +539,17 @@ static void parse_command_line(int argc, const char* argv[], pack_opt_t* options break; + + case 'u': + + options->ublock_filename = opt_arg; + break; + + case 'b': + + options->ublock_size = atoi( opt_arg ); + break; + } /* switch */ diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index 3796179..e7024f2 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -71,6 +71,9 @@ #define FNAME16 "h5repack_ub.h5" #define FNAME16OUT "h5repack_ub_out.h5" +#define FNAME_UB "ublock.bin" + + const char *H5REPACK_FILENAMES[] = { "h5repack_big_out", @@ -123,7 +126,8 @@ int make_attr(hid_t loc_id,int rank,hsize_t *dims,const char *attr_name,hid_t ty void make_dset_reg_ref(hid_t loc_id); int make_external(hid_t loc_id); static int make_userblock(void); -static int verify_userblock(void); +static int verify_userblock( const char* filename); +static int make_userblock_file(void); /*------------------------------------------------------------------------- @@ -1332,7 +1336,7 @@ if (szip_can_encode) { GOERROR; if(h5repack_verify(FNAME16OUT, &pack_options) <= 0) GOERROR; - if(verify_userblock() < 0) + if(verify_userblock(FNAME16OUT) < 0) GOERROR; if(h5repack_end(&pack_options) < 0) GOERROR; @@ -1394,6 +1398,32 @@ if (szip_can_encode) { #endif + /*------------------------------------------------------------------------- + * test file with userblock + *------------------------------------------------------------------------- + */ + TESTING(" file with added userblock"); + if(h5repack_init(&pack_options, 0) < 0) + GOERROR; + + /* add the options for a user block size and user block filename */ + pack_options.ublock_size = USERBLOCK_SIZE; + pack_options.ublock_filename = FNAME_UB; + + if(h5repack(FNAME8, FNAME8OUT, &pack_options) < 0) + GOERROR; + if(h5diff(FNAME8, FNAME8OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(FNAME8OUT, &pack_options) <= 0) + GOERROR; + if(verify_userblock(FNAME8OUT) < 0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + + /*------------------------------------------------------------------------- * clean temporary test files *------------------------------------------------------------------------- @@ -1600,6 +1630,13 @@ int make_testfiles(void) if(make_userblock() < 0) goto out; + /*------------------------------------------------------------------------- + * create a userblock file + *------------------------------------------------------------------------- + */ + if(make_userblock_file() < 0) + goto out; + return 0; out: @@ -2879,7 +2916,7 @@ out: *------------------------------------------------------------------------- */ static int -verify_userblock(void) +verify_userblock( const char* filename) { hid_t fid = -1; hid_t fcpl = -1; @@ -2890,7 +2927,7 @@ verify_userblock(void) size_t u; /* Local index variable */ /* Open file with userblock */ - if((fid = H5Fopen(FNAME16OUT, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + if((fid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) goto out; /* Retrieve file creation property list & userblock size */ @@ -2912,7 +2949,7 @@ verify_userblock(void) /* Re-open HDF5 file, as "plain" file */ - if((fd = HDopen(FNAME16, O_RDONLY, 0)) < 0) + if((fd = HDopen(filename, O_RDONLY, 0)) < 0) goto out; /* Read userblock data */ @@ -2942,6 +2979,48 @@ out: /*------------------------------------------------------------------------- + * Function: make_userblock_file + * + * Purpose: create a file for the userblock add test + * + *------------------------------------------------------------------------- + */ +static int +make_userblock_file(void) +{ + hid_t fid = -1; + hid_t fcpl = -1; + int fd = -1; /* File descriptor for writing userblock */ + char ub[USERBLOCK_SIZE]; /* User block data */ + ssize_t nwritten; /* # of bytes written */ + size_t u; /* Local index variable */ + + /* initialize userblock data */ + for(u = 0; u < USERBLOCK_SIZE; u++) + ub[u] = 'a' + (u % 26); + + /* open file */ + if((fd = HDopen(FNAME_UB,O_WRONLY|O_CREAT|O_TRUNC, 0644 )) < 0) + goto out; + + /* write userblock data */ + nwritten = HDwrite(fd, ub, (size_t)USERBLOCK_SIZE); + assert(nwritten == USERBLOCK_SIZE); + + /* close file */ + HDclose(fd); + + return 0; + +out: + + if(fd > 0) + HDclose(fd); + + return -1; +} + +/*------------------------------------------------------------------------- * Function: write_dset_in * * Purpose: write datasets in LOC_ID -- cgit v0.12