diff options
Diffstat (limited to 'tools')
24 files changed, 776 insertions, 227 deletions
diff --git a/tools/h5dump/h5dump.c b/tools/h5dump/h5dump.c index 43b2813..a437619 100644 --- a/tools/h5dump/h5dump.c +++ b/tools/h5dump/h5dump.c @@ -3067,22 +3067,22 @@ dump_fcpl(hid_t fid) hsize_t userblock; /* userblock size retrieved from FCPL */ size_t off_size; /* size of offsets in the file */ size_t len_size; /* size of lengths in the file */ - unsigned super; /* superblock version # */ - unsigned freelist; /* free list version # */ - unsigned stab; /* symbol table entry version # */ - unsigned shhdr; /* shared object header version # */ hid_t fdriver; /* file driver */ char dname[32]; /* buffer to store driver name */ unsigned sym_lk; /* symbol table B-tree leaf 'K' value */ unsigned sym_ik; /* symbol table B-tree internal 'K' value */ unsigned istore_ik; /* indexed storage B-tree internal 'K' value */ + H5F_file_space_type_t fs_strategy; /* file space strategy */ + hsize_t fs_threshold; /* free-space section threshold */ + H5F_info2_t finfo; /* file information */ fcpl=H5Fget_create_plist(fid); - H5Pget_version(fcpl, &super, &freelist, &stab, &shhdr); + H5Fget_info2(fid, &finfo); H5Pget_userblock(fcpl,&userblock); H5Pget_sizes(fcpl,&off_size,&len_size); H5Pget_sym_k(fcpl,&sym_ik,&sym_lk); H5Pget_istore_k(fcpl,&istore_ik); + H5Pget_file_space(fcpl, &fs_strategy, &fs_threshold); H5Pclose(fcpl); fapl=h5_fileaccess(); fdriver=H5Pget_driver(fapl); @@ -3094,13 +3094,13 @@ dump_fcpl(hid_t fid) */ printf("%s %s\n",SUPER_BLOCK, BEGIN); indentation(indent + COL); - printf("%s %u\n","SUPERBLOCK_VERSION", super); + printf("%s %u\n","SUPERBLOCK_VERSION", finfo.super.version); indentation(indent + COL); - printf("%s %u\n","FREELIST_VERSION", freelist); + printf("%s %u\n","FREELIST_VERSION", finfo.free.version); indentation(indent + COL); - printf("%s %u\n","SYMBOLTABLE_VERSION", stab); + printf("%s %u\n","SYMBOLTABLE_VERSION", 0); /* Retain this for backward compatibility, for now (QAK) */ indentation(indent + COL); - printf("%s %u\n","OBJECTHEADER_VERSION", shhdr); + printf("%s %u\n","OBJECTHEADER_VERSION", finfo.sohm.version); indentation(indent + COL); HDfprintf(stdout,"%s %Hd\n","OFFSET_SIZE", (long long)off_size); indentation(indent + COL); @@ -3141,6 +3141,21 @@ dump_fcpl(hid_t fid) printf("%s %s\n","FILE_DRIVER", dname);*/ indentation(indent + COL); printf("%s %u\n","ISTORE_K", istore_ik); + + indentation(indent + COL); + if(fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) + printf("%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_ALL_PERSIST"); + else if(fs_strategy == H5F_FILE_SPACE_ALL) + printf("%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_ALL"); + else if(fs_strategy == H5F_FILE_SPACE_AGGR_VFD) + printf("%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_AGGR_VFD"); + else if(fs_strategy == H5F_FILE_SPACE_VFD) + printf("%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_VFD"); + else + printf("%s %s\n", "FILE_SPACE_STRATEGY", "Unknown strategy"); + indentation(indent + COL); + HDfprintf(stdout, "%s %Hu\n","FREE_SPACE_THRESHOLD", fs_threshold); + printf("%s\n",END); /*------------------------------------------------------------------------- diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c index b8ed41d..2ddc0bf 100644 --- a/tools/h5dump/h5dumpgentest.c +++ b/tools/h5dump/h5dumpgentest.c @@ -93,6 +93,7 @@ #define FILE63 "textlinkfar.h5" #define FILE64 "tarray8.h5" #define FILE65 "tattrreg.h5" +#define FILE66 "file_space.h5" @@ -244,6 +245,10 @@ typedef struct s1_t { #define F64_ARRAY_BUF_LEN (4*1024) #define F64_DIM1 (F64_ARRAY_BUF_LEN / sizeof(int) + 1) +/* File 65 macros */ +#define STRATEGY H5F_FILE_SPACE_AGGR_VFD /* File space handling strategy */ +#define THRESHOLD10 10 /* Free space section threshold */ + static void gent_group(void) { @@ -6443,7 +6448,32 @@ gent_extlinks(void) H5Fclose(far_fid); } +/*------------------------------------------------------------------------- + * Function: gent_fs_strategy_threshold + * + * Purpose: Generate a file with non-default file space strategy and + * non-default free-space section threshold. + *------------------------------------------------------------------------- + */ +static void +gent_fs_strategy_threshold(void) +{ + hid_t fid; /* File id */ + hid_t fcpl; /* File creation property */ + + /* Create file-creation template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + + /* Set file space information */ + H5Pset_file_space(fcpl, STRATEGY, THRESHOLD10); + /* Create the file with the specified strategy and threshold */ + fid = H5Fcreate(FILE66, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT); + + /* close */ + H5Fclose(fid); + H5Pclose(fcpl); +} /*------------------------------------------------------------------------- * Function: main @@ -6516,7 +6546,7 @@ int main(void) gent_attr_creation_order(); gent_fpformat(); gent_extlinks(); - + gent_fs_strategy_threshold(); return 0; } diff --git a/tools/h5dump/testh5dump.sh.in b/tools/h5dump/testh5dump.sh.in index 3ffff50..9dcb562 100644 --- a/tools/h5dump/testh5dump.sh.in +++ b/tools/h5dump/testh5dump.sh.in @@ -340,6 +340,7 @@ TOOLTEST tnofilename.ddl # tests for super block TOOLTEST tboot1.ddl -H -B -d dset tfcontents1.h5 TOOLTEST tboot2.ddl -B tfcontents2.h5 +TOOLTEST file_space.ddl -B file_space.h5 # test -p with a non existing dataset TOOLTEST tperror.ddl -p -d bogus tfcontents1.h5 diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index d8a503c..e6b95a0 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -90,8 +90,8 @@ int h5repack(const char* infile, *------------------------------------------------------------------------- */ -int h5repack_init (pack_opt_t *options, - int verbose) +int +h5repack_init(pack_opt_t *options, int verbose, H5F_file_space_type_t strategy, hsize_t threshold) { int k, n; memset(options,0,sizeof(pack_opt_t)); @@ -106,6 +106,9 @@ int h5repack_init (pack_opt_t *options, options->filter_g[n].cd_values[k] = 0; } + options->fs_strategy = strategy; + options->fs_threshold = threshold; + return (options_table_init(&(options->op_tbl))); } diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index cb3d75a..16f438c 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -112,6 +112,8 @@ typedef struct { hsize_t ublock_size; /* user block size */ hsize_t threshold; /* alignment threshold for H5Pset_alignment */ hsize_t alignment ; /* alignment for H5Pset_alignment */ + H5F_file_space_type_t fs_strategy; /* File space handling strategy */ + hsize_t fs_threshold; /* Free space section threshold */ } pack_opt_t; @@ -125,14 +127,13 @@ typedef struct { extern "C" { #endif -int h5repack (const char* infile, const char* outfile, pack_opt_t *options); -int h5repack_addfilter (const char* str, pack_opt_t *options); -int h5repack_addlayout (const char* str, pack_opt_t *options); -int h5repack_init (pack_opt_t *options, int verbose); -int h5repack_end (pack_opt_t *options); -int h5repack_verify (const char *fname,pack_opt_t *options); -int h5repack_cmp_pl (const char *fname1, - const char *fname2); +int h5repack(const char* infile, const char* outfile, pack_opt_t *options); +int h5repack_addfilter(const char* str, pack_opt_t *options); +int h5repack_addlayout(const char* str, pack_opt_t *options); +int h5repack_init(pack_opt_t *options, int verbose, H5F_file_space_type_t strategy, hsize_t threshold); +int h5repack_end(pack_opt_t *options); +int h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options); +int h5repack_cmp_pl(const char *fname1, const char *fname2); #ifdef __cplusplus diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 0fb8fc8..77c03d2 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -104,7 +104,7 @@ int copy_objects(const char* fnamein, goto out; } - /* get user block size */ + /* get user block size and file space strategy/threshold */ { hid_t fcpl_in; /* file creation property list ID for input file */ @@ -120,6 +120,24 @@ int copy_objects(const char* fnamein, goto out; } + if(!options->fs_strategy) + { + if(H5Pget_file_space(fcpl_in, &options->fs_strategy, NULL) < 0) + { + error_msg(progname, "failed to retrieve file space strategy\n"); + goto out; + } + } + + if(!options->fs_threshold) + { + if(H5Pget_file_space(fcpl_in, NULL, &options->fs_threshold) < 0) + { + error_msg(progname, "failed to retrieve file space threshold\n"); + goto out; + } + } + if(H5Pclose(fcpl_in) < 0) { error_msg(progname, "failed to close property list\n"); @@ -316,6 +334,32 @@ int copy_objects(const char* fnamein, } + /* either use the FCPL already created or create a new one */ + if(fcpl != H5P_DEFAULT) + { + /* set file space strategy and free space threshold */ + if(H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) + { + error_msg(progname, "failed to set file space strategy & threshold\n"); + goto out; + } + } + else + { + /* create a file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + { + error_msg(progname, "fail to create a file creation property list\n"); + goto out; + } + + /* set file space strategy and free space threshold */ + if(H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) + { + error_msg(progname, "failed to set file space strategy & threshold \n"); + goto out; + } + } /*------------------------------------------------------------------------- * create the output file diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 63aba1f..b6e645a3 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -38,7 +38,7 @@ const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:t:a:i:o:"; +static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:t:a:i:o:S:T:"; static struct long_options l_opts[] = { { "help", no_arg, 'h' }, { "version", no_arg, 'V' }, @@ -58,6 +58,8 @@ static struct long_options l_opts[] = { { "alignment", require_arg, 'a' }, { "infile", require_arg, 'i' }, /* -i for backward compability */ { "outfile", require_arg, 'o' }, /* -o for backward compability */ + { "fs_strategy", require_arg, 'S' }, + { "fs_threshold", require_arg, 'T' }, { NULL, 0, '\0' } }; @@ -104,7 +106,7 @@ int main(int argc, const char **argv) int ret=-1; /* initialize options */ - h5repack_init (&options,0); + h5repack_init (&options, 0, 0, (hsize_t)0); parse_command_line(argc, argv, &options); @@ -180,6 +182,8 @@ static void usage(const char *prog) printf(" -a A, --alignment=A Alignment value for H5Pset_alignment\n"); printf(" -f FILT, --filter=FILT Filter type\n"); printf(" -l LAYT, --layout=LAYT Layout type\n"); + printf(" -S STRGY, --fs_strategy=STRGY File-space-handling strategy\n"); + printf(" -T THRD, --fs_threshold=THRD Free-space section threshold\n"); printf("\n"); @@ -193,7 +197,18 @@ static void usage(const char *prog) printf(" a power of 2 (1024 default)\n"); printf(" F - is the shared object header message type, any of <dspace|dtype|fill|\n"); printf(" pline|attr>. If F is not specified, S applies to all messages\n"); - + printf("\n"); + printf(" STRGY is an integer value listed below:\n"); + printf(" 1: Use persistent free-space managers, aggregators and virtual file driver for allocation\n"); + printf(" 2: Use non-persistent free-space managers, aggregators and virtual file driver for allocation\n"); + printf(" 3: Use aggregators and virtual file driver for allocation\n"); + printf(" 4: Use virtual file driver for allocation\n"); + printf(" A value of 0 retains the existing file-space-handling strategy used in the library.\n"); + printf(" The library default is 2.\n"); + printf("\n"); + printf(" THRD is the minimum size of free-space sections to track by the free-space managers.\n"); + printf(" A value of 0 retains the existing free-space section threshold used in the library.\n"); + printf(" The library default is 1.\n"); printf("\n"); printf(" FILT - is a string with the format:\n"); @@ -429,6 +444,15 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options) } break; + case 'S': + + options->fs_strategy = atol( opt_arg ); + break; + + case 'T': + + options->fs_threshold = (hsize_t)atol( opt_arg ); + break; } /* switch */ diff --git a/tools/h5repack/h5repack_verify.c b/tools/h5repack/h5repack_verify.c index 83985c9..fe94148 100644 --- a/tools/h5repack/h5repack_verify.c +++ b/tools/h5repack/h5repack_verify.c @@ -44,20 +44,25 @@ static int verify_filters(hid_t pid, hid_t tid, int nfilters, filter_info_t *fil *------------------------------------------------------------------------- */ -int h5repack_verify(const char *fname, - pack_opt_t *options) +int +h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options) { - hid_t fid; /* file ID */ - hid_t did; /* dataset ID */ - hid_t pid; /* dataset creation property list ID */ - hid_t sid; /* space ID */ - hid_t tid; /* type ID */ + hid_t fidin; /* file ID for input file*/ + hid_t fidout; /* file ID for output file*/ + hid_t did; /* dataset ID */ + hid_t pid; /* dataset creation property list ID */ + hid_t sid; /* space ID */ + hid_t tid; /* type ID */ unsigned int i; trav_table_t *travt = NULL; int ok = 1; + hid_t fcpl_in; /* file creation property for input file */ + hid_t fcpl_out; /* file creation property for output file */ + H5F_file_space_type_t in_strat, out_strat; /* file space handling strategy for in/output file */ + hsize_t in_thresh, out_thresh; /* free space section threshold for in/output file */ - /* open the file */ - if((fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) + /* open the output file */ + if((fidout = H5Fopen(out_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) return -1; for(i = 0; i < options->op_tbl->nelems; i++) @@ -69,7 +74,7 @@ int h5repack_verify(const char *fname, * open *------------------------------------------------------------------------- */ - if((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) + if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0) goto error; if((sid = H5Dget_space(did)) < 0) goto error; @@ -121,7 +126,7 @@ int h5repack_verify(const char *fname, trav_table_init(&travt); /* get the list of objects in the file */ - if(h5trav_gettable(fid, travt) < 0) + if(h5trav_gettable(fidout, travt) < 0) goto error; for(i = 0; i < travt->nobjs; i++) @@ -135,7 +140,7 @@ int h5repack_verify(const char *fname, * open *------------------------------------------------------------------------- */ - if((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) + if((did = H5Dopen2(fidout, name, H5P_DEFAULT)) < 0) goto error; if((sid = H5Dget_space(did)) < 0) goto error; @@ -191,21 +196,95 @@ int h5repack_verify(const char *fname, } /*------------------------------------------------------------------------- - * close + * Verify that file space strategy and free space threshold + * are set as expected *------------------------------------------------------------------------- */ - if (H5Fclose(fid) < 0) - return -1; + /* open the input file */ + if((fidin = H5Fopen(in_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) + goto error; + + /* Get file creation property list for input file */ + if((fcpl_in = H5Fget_create_plist(fidin)) < 0) { + error_msg(progname, "failed to retrieve file creation property list\n"); + goto error; + } + + /* Get file space management info for input file */ + if(H5Pget_file_space(fcpl_in, &in_strat, &in_thresh) < 0) { + error_msg(progname, "failed to retrieve file space strategy & threshold\n"); + goto error; + } + + /* Output file is already opened */ + /* Get file creation property list for output file */ + if((fcpl_out = H5Fget_create_plist(fidout)) < 0) { + error_msg(progname, "failed to retrieve file creation property list\n"); + goto error; + } + + /* Get file space management info for output file */ + if(H5Pget_file_space(fcpl_out, &out_strat, &out_thresh) < 0) { + error_msg(progname, "failed to retrieve file space strategy & threshold\n"); + goto error; + } + + /* + * If the strategy option is not set, + * file space handling strategy should be the same for both + * input & output files. + * If the strategy option is set, + * the output file's file space handling strategy should be the same + * as what is set via the strategy option + */ + if(!options->fs_strategy && out_strat != in_strat) { + error_msg(progname, "file space strategy not set as unexpected\n"); + goto error; + + } else if(options->fs_strategy && out_strat!= options->fs_strategy) { + error_msg(progname, "file space strategy not set as unexpectec\n"); + goto error; + } + + /* + * If the threshold option is not set, + * the free space section threshold should be the same for both + * input & output files. + * If the threshold option is set, + * the output file's free space section threshold should be the same + * as what is set via the threshold option. + */ + if(!options->fs_threshold && out_thresh != in_thresh) { + error_msg(progname, "free space threshold not set as unexpected\n"); + goto error; + + } else if(options->fs_threshold && out_thresh != options->fs_threshold) { + error_msg(progname, "free space threshold not set as unexpectec\n"); + goto error; + } + + /* Closing */ + if (H5Pclose(fcpl_in) < 0) + goto error; + if (H5Pclose(fcpl_out) < 0) + goto error; + if (H5Fclose(fidin) < 0) + goto error; + if (H5Fclose(fidout) < 0) + goto error; return ok; error: H5E_BEGIN_TRY { + H5Pclose(fcpl_in); + H5Pclose(fcpl_out); H5Pclose(pid); H5Sclose(sid); H5Dclose(did); - H5Fclose(fid); + H5Fclose(fidin); + H5Fclose(fidout); if (travt) trav_table_free(travt); } H5E_END_TRY; diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index 7ac339a..e36a8dc 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -152,6 +152,8 @@ int main (void) { pack_opt_t pack_options; diff_opt_t diff_options; + hsize_t fs_size = 0; /* free space section threshold */ + H5F_file_space_type_t fs_type = H5F_FILE_SPACE_DEFAULT; /* file space handling strategy */ #if defined (H5_HAVE_FILTER_SZIP) int szip_can_encode = 0; #endif @@ -184,13 +186,15 @@ int main (void) */ TESTING(" copy of datasets (fill values)"); - if (h5repack_init (&pack_options, 0) < 0) + + /* fs_type = 0; fs_size = 0 */ + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME0,FNAME0OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME0,FNAME0OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME0OUT,&pack_options)<=0) + if (h5repack_verify(FNAME0, FNAME0OUT,&pack_options)<=0) GOERROR; if (h5repack_cmp_pl(FNAME0,FNAME0OUT)<=0) GOERROR; @@ -204,13 +208,13 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (all datatypes)"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME1,FNAME1OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME1,FNAME1OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME1OUT,&pack_options)<=0) + if (h5repack_verify(FNAME1, FNAME1OUT,&pack_options)<=0) GOERROR; if (h5repack_cmp_pl(FNAME1,FNAME1OUT)<=0) GOERROR; @@ -224,13 +228,13 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (attributes)"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME2,FNAME2OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME2,FNAME2OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME2OUT,&pack_options)<=0) + if (h5repack_verify(FNAME2, FNAME2OUT,&pack_options)<=0) GOERROR; if (h5repack_cmp_pl(FNAME2,FNAME2OUT)<=0) GOERROR; @@ -243,13 +247,13 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (hardlinks)"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME3,FNAME3OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME3,FNAME3OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME3OUT,&pack_options)<=0) + if (h5repack_verify(FNAME3, FNAME3OUT,&pack_options)<=0) GOERROR; if (h5repack_cmp_pl(FNAME3,FNAME3OUT)<=0) GOERROR; @@ -263,13 +267,13 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of allocation early file"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME5,FNAME5OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME5,FNAME5OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME5OUT,&pack_options)<=0) + if (h5repack_verify(FNAME5, FNAME5OUT, &pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -293,7 +297,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset1:GZIP=9",&pack_options) < 0) GOERROR; @@ -303,7 +307,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -321,7 +325,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -331,7 +335,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -359,7 +363,7 @@ int main (void) */ if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset2:SZIP=8,EC",&pack_options) < 0) GOERROR; @@ -369,7 +373,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -391,7 +395,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -399,7 +403,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -422,7 +426,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset1:SHUF",&pack_options) < 0) GOERROR; @@ -432,7 +436,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -451,7 +455,8 @@ int main (void) #ifdef H5_HAVE_FILTER_SHUFFLE - if (h5repack_init (&pack_options, 0) < 0) + /* fs_type = H5F_FILE_SPACE_ALL_PERSIST; fs_size = 1 */ + if (h5repack_init (&pack_options, 0, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) GOERROR; if (h5repack_addfilter("SHUF",&pack_options) < 0) GOERROR; @@ -461,7 +466,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -480,7 +485,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset1:FLET",&pack_options) < 0) GOERROR; @@ -490,7 +495,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -510,7 +515,7 @@ int main (void) #ifdef H5_HAVE_FILTER_FLETCHER32 - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("FLET",&pack_options) < 0) GOERROR; @@ -520,7 +525,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -538,7 +543,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options) < 0) GOERROR; @@ -569,7 +574,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -584,7 +589,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -592,7 +597,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -604,7 +609,7 @@ int main (void) */ TESTING(" adding layout chunked to all"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -612,7 +617,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -625,7 +630,7 @@ int main (void) * test an individual object option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset1:CONTI",&pack_options) < 0) GOERROR; @@ -633,7 +638,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -646,7 +651,7 @@ int main (void) * test all objects option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -654,7 +659,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -663,7 +668,7 @@ int main (void) * do the same test for a file with filters (chunked) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -671,7 +676,7 @@ int main (void) GOERROR; if (h5diff(FNAME8,FNAME8OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME8OUT,&pack_options)<=0) + if (h5repack_verify(FNAME8, FNAME8OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -685,7 +690,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset1:COMPA",&pack_options) < 0) GOERROR; @@ -693,7 +698,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -706,7 +711,8 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + /* fs_type = H5F_FILE_SPACE_ALL; fs_size = 2 */ + if (h5repack_init (&pack_options, 0, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) GOERROR; if (h5repack_addlayout("COMPA",&pack_options) < 0) GOERROR; @@ -714,7 +720,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -728,7 +734,7 @@ int main (void) * layout compact to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CONTI",&pack_options) < 0) GOERROR; @@ -736,7 +742,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -748,7 +754,7 @@ int main (void) * layout compact to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CHUNK=2x5",&pack_options) < 0) GOERROR; @@ -756,7 +762,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -768,7 +774,7 @@ int main (void) * layout compact to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_compact:COMPA",&pack_options) < 0) GOERROR; @@ -776,7 +782,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -787,7 +793,7 @@ int main (void) * layout contiguous to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:COMPA",&pack_options) < 0) GOERROR; @@ -795,7 +801,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -806,7 +812,7 @@ int main (void) * layout contiguous to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CHUNK=3x6",&pack_options) < 0) GOERROR; @@ -814,7 +820,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -826,7 +832,7 @@ int main (void) * layout contiguous to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CONTI",&pack_options) < 0) GOERROR; @@ -834,7 +840,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -845,7 +851,7 @@ int main (void) * layout chunked to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:COMPA",&pack_options) < 0) GOERROR; @@ -853,7 +859,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -865,7 +871,7 @@ int main (void) * layout chunked to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CONTI",&pack_options) < 0) GOERROR; @@ -873,7 +879,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -884,7 +890,7 @@ int main (void) * layout chunked to chunked conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CHUNK=18x13",&pack_options) < 0) GOERROR; @@ -892,7 +898,7 @@ int main (void) GOERROR; if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + if (h5repack_verify(FNAME4, FNAME4OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -914,13 +920,14 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + /* fs_type = H5F_FILE_SPACE_AGGR_VFD; fs_size = 3 */ + if (h5repack_init (&pack_options, 0, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) GOERROR; if (h5repack(FNAME7,FNAME7OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME7,FNAME7OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME7OUT,&pack_options)<=0) + if (h5repack_verify(FNAME7, FNAME7OUT,&pack_options)<=0) GOERROR; if (h5repack_cmp_pl(FNAME7,FNAME7OUT)<=0) GOERROR; @@ -939,7 +946,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_szip:NONE",&pack_options) < 0) GOERROR; @@ -947,7 +954,7 @@ int main (void) GOERROR; if (h5diff(FNAME7,FNAME7OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME7OUT,&pack_options)<=0) + if (h5repack_verify(FNAME7, FNAME7OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -964,13 +971,13 @@ int main (void) TESTING(" copy of deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME8,FNAME8OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME8,FNAME8OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME8OUT,&pack_options)<=0) + if (h5repack_verify(FNAME8, FNAME8OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -984,7 +991,7 @@ int main (void) TESTING(" removing deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, ++fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:NONE",&pack_options) < 0) GOERROR; @@ -992,7 +999,7 @@ int main (void) GOERROR; if (h5diff(FNAME8,FNAME8OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME8OUT,&pack_options)<=0) + if (h5repack_verify(FNAME8, FNAME8OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1007,13 +1014,13 @@ int main (void) TESTING(" copy of shuffle filter"); #ifdef H5_HAVE_FILTER_SHUFFLE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME9,FNAME9OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME9,FNAME9OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME9OUT,&pack_options)<=0) + if (h5repack_verify(FNAME9, FNAME9OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1026,7 +1033,7 @@ int main (void) TESTING(" removing shuffle filter"); #ifdef H5_HAVE_FILTER_SHUFFLE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_shuffle:NONE",&pack_options) < 0) GOERROR; @@ -1034,7 +1041,7 @@ int main (void) GOERROR; if (h5diff(FNAME9,FNAME9OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME9OUT,&pack_options)<=0) + if (h5repack_verify(FNAME9, FNAME9OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1047,13 +1054,13 @@ int main (void) TESTING(" copy of fletcher filter"); #ifdef H5_HAVE_FILTER_FLETCHER32 - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME10,FNAME10OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME10,FNAME10OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME10OUT,&pack_options)<=0) + if (h5repack_verify(FNAME10, FNAME10OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1066,7 +1073,7 @@ int main (void) TESTING(" removing fletcher filter"); #ifdef H5_HAVE_FILTER_FLETCHER32 - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_fletcher32:NONE",&pack_options) < 0) GOERROR; @@ -1074,7 +1081,7 @@ int main (void) GOERROR; if (h5diff(FNAME10,FNAME10OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME10OUT,&pack_options)<=0) + if (h5repack_verify(FNAME10, FNAME10OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1088,13 +1095,13 @@ int main (void) TESTING(" copy of nbit filter"); #ifdef H5_HAVE_FILTER_NBIT - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME12,FNAME12OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME12,FNAME12OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME12OUT,&pack_options)<=0) + if (h5repack_verify(FNAME12, FNAME12OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1107,7 +1114,7 @@ int main (void) TESTING(" removing nbit filter"); #ifdef H5_HAVE_FILTER_NBIT - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_nbit:NONE",&pack_options) < 0) GOERROR; @@ -1115,7 +1122,7 @@ int main (void) GOERROR; if (h5diff(FNAME12,FNAME12OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME12OUT,&pack_options)<=0) + if (h5repack_verify(FNAME12, FNAME12OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1129,7 +1136,7 @@ int main (void) TESTING(" adding nbit filter"); #ifdef H5_HAVE_FILTER_NBIT - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_int31:NBIT",&pack_options) < 0) GOERROR; @@ -1137,7 +1144,7 @@ int main (void) GOERROR; if (h5diff(FNAME12,FNAME12OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME12OUT,&pack_options)<=0) + if (h5repack_verify(FNAME12, FNAME12OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1151,13 +1158,13 @@ int main (void) TESTING(" copy of scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME13,FNAME13OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME13OUT,&pack_options)<=0) + if (h5repack_verify(FNAME13, FNAME13OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1170,7 +1177,7 @@ int main (void) TESTING(" removing scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_scaleoffset:NONE",&pack_options) < 0) GOERROR; @@ -1178,7 +1185,7 @@ int main (void) GOERROR; if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME13OUT,&pack_options)<=0) + if (h5repack_verify(FNAME13, FNAME13OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1192,7 +1199,7 @@ int main (void) TESTING(" adding scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_none:SOFF=31,IN",&pack_options) < 0) GOERROR; @@ -1200,7 +1207,7 @@ int main (void) GOERROR; if (h5diff(FNAME13,FNAME13OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME13OUT,&pack_options)<=0) + if (h5repack_verify(FNAME13, FNAME13OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1230,7 +1237,8 @@ int main (void) && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + /* fs_type = H5F_FILE_SPACE_VFD; fs_size = 4 */ + if (h5repack_init (&pack_options, 0, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -1238,7 +1246,7 @@ int main (void) GOERROR; if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME11OUT,&pack_options)<=0) + if (h5repack_verify(FNAME11, FNAME11OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1258,7 +1266,7 @@ int main (void) && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options) < 0) GOERROR; @@ -1266,7 +1274,7 @@ int main (void) GOERROR; if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME11OUT,&pack_options)<=0) + if (h5repack_verify(FNAME11, FNAME11OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1290,7 +1298,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) \ && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("NONE",&pack_options) < 0) GOERROR; @@ -1298,7 +1306,7 @@ int main (void) GOERROR; if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME11OUT,&pack_options)<=0) + if (h5repack_verify(FNAME11, FNAME11OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1313,13 +1321,14 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" big file"); - if (h5repack_init (&pack_options, 0) < 0) + + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME14,FNAME14OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME14,FNAME14OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME14OUT,&pack_options)<=0) + if (h5repack_verify(FNAME14, FNAME14OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1330,13 +1339,13 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" external datasets"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack(FNAME15,FNAME15OUT,&pack_options) < 0) GOERROR; if (h5diff(FNAME15,FNAME15OUT,NULL,NULL,&diff_options) > 0) GOERROR; - if (h5repack_verify(FNAME15OUT,&pack_options)<=0) + if (h5repack_verify(FNAME15, FNAME15OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1347,13 +1356,13 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" file with userblock"); - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if(h5repack(FNAME16, FNAME16OUT, &pack_options) < 0) GOERROR; if(h5diff(FNAME16, FNAME16OUT, NULL, NULL, &diff_options) > 0) GOERROR; - if(h5repack_verify(FNAME16OUT, &pack_options) <= 0) + if(h5repack_verify(FNAME16, FNAME16OUT, &pack_options) <= 0) GOERROR; if(verify_userblock(FNAME16OUT) < 0) GOERROR; @@ -1366,7 +1375,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" latest file format options"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; pack_options.latest=1; pack_options.grp_compact=10; @@ -1380,7 +1389,7 @@ int main (void) GOERROR; if (h5diff(FNAME1,FNAME1OUT,NULL,NULL,&diff_options) > 0) GOERROR; - if (h5repack_verify(FNAME1OUT,&pack_options)<=0) + if (h5repack_verify(FNAME1, FNAME1OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1396,7 +1405,7 @@ int main (void) #if defined (H5_HAVE_FILTER_DEFLATE) && defined (H5_HAVE_FILTER_SHUFFLE) - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -1406,7 +1415,7 @@ int main (void) GOERROR; if (h5diff(FNAME11,FNAME11OUT,NULL,NULL,&diff_options) >0) GOERROR; - if (h5repack_verify(FNAME11OUT,&pack_options)<=0) + if (h5repack_verify(FNAME11, FNAME11OUT,&pack_options)<=0) GOERROR; if (h5repack_end (&pack_options) < 0) GOERROR; @@ -1425,7 +1434,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, fs_type, fs_size) < 0) GOERROR; /* add the options for a user block size and user block filename */ @@ -1436,7 +1445,7 @@ int main (void) GOERROR; if(h5diff(FNAME8, FNAME8OUT, NULL, NULL, &diff_options) > 0) GOERROR; - if(h5repack_verify(FNAME8OUT, &pack_options) <= 0) + if(h5repack_verify(FNAME8, FNAME8OUT, &pack_options) <= 0) GOERROR; if(verify_userblock(FNAME8OUT) < 0) GOERROR; @@ -1458,7 +1467,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, fs_type, fs_size) < 0) GOERROR; /* add the options for aligment */ @@ -1469,7 +1478,7 @@ int main (void) GOERROR; if(h5diff(FNAME8, FNAME8OUT, NULL, NULL, &diff_options) > 0) GOERROR; - if(h5repack_verify(FNAME8OUT, &pack_options) <= 0) + if(h5repack_verify(FNAME8, FNAME8OUT, &pack_options) <= 0) GOERROR; /* verify aligment */ @@ -1512,14 +1521,14 @@ int main (void) */ TESTING(" file with committed datatypes"); - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, fs_type, fs_size) < 0) GOERROR; if(h5repack(FNAME17, FNAME17OUT, &pack_options) < 0) GOERROR; if(h5diff(FNAME17, FNAME17OUT, NULL, NULL, &diff_options) > 0) GOERROR; - if(h5repack_verify(FNAME17OUT, &pack_options) <= 0) + if(h5repack_verify(FNAME17, FNAME17OUT, &pack_options) <= 0) GOERROR; if(h5repack_end(&pack_options) < 0) GOERROR; diff --git a/tools/h5stat/h5stat.c b/tools/h5stat/h5stat.c index f5537e1..30ad0a1 100644 --- a/tools/h5stat/h5stat.c +++ b/tools/h5stat/h5stat.c @@ -26,6 +26,7 @@ #define SIZE_SMALL_GROUPS 10 #define SIZE_SMALL_ATTRS 10 #define SIZE_SMALL_DSETS 10 +#define SIZE_SMALL_SECTS 10 #define H5_NFILTERS_IMPL 8 /* Number of currently implemented filters + one to accommodate for user-define filters + one @@ -48,7 +49,7 @@ typedef struct ohdr_info_t { /* Info to pass to the iteration functions */ typedef struct iter_t { hid_t fid; /* File ID */ - + hsize_t filesize; /* Size of the file */ unsigned long uniq_groups; /* Number of unique groups */ unsigned long uniq_dsets; /* Number of unique datasets */ unsigned long uniq_dtypes; /* Number of unique named datatypes */ @@ -79,6 +80,7 @@ typedef struct iter_t { unsigned long *dset_dim_bins; /* Pointer to array of bins for dataset dimensions */ ohdr_info_t dset_ohdr_info; /* Object header information for datasets */ hsize_t dset_storage_size; /* Size of raw data for datasets */ + hsize_t dset_external_storage_size; /* Size of raw data for datasets with external storage */ ohdr_info_t dtype_ohdr_info; /* Object header information for datatypes */ hsize_t groups_btree_storage_size; /* btree size for group */ hsize_t groups_heap_storage_size; /* heap size for group */ @@ -87,8 +89,16 @@ typedef struct iter_t { hsize_t SM_hdr_storage_size; /* header size for SOHM table (1.8) */ hsize_t SM_index_storage_size; /* index (btree & list) size for SOHM table (1.8) */ hsize_t SM_heap_storage_size; /* fractal heap size for SOHM table (1.8) */ + hsize_t super_size; /* superblock size */ hsize_t super_ext_size; /* superblock extension size */ + hsize_t ublk_size; /* user block size (if exists) */ + hsize_t free_space; /* amount of freespace in the file */ + hsize_t free_hdr; /* free space manager header in the file */ + unsigned long num_small_sects[SIZE_SMALL_SECTS]; /* Size of small free-space sections */ + unsigned sect_nbins; /* Number of bins for free-space section sizes */ + unsigned long *sect_bins; /* Pointer to array of bins for free-space section sizes */ hsize_t datasets_index_storage_size;/* meta size for chunked dataset's indexing type */ + hsize_t datasets_heap_storage_size; /* heap size for dataset with external storage */ unsigned long nexternal; /* Number of external files for a dataset */ int local; /* Flag to indicate iteration over the object*/ } iter_t; @@ -104,6 +114,8 @@ static int display_dset = FALSE; static int display_dtype_metadata = FALSE; static int display_object = FALSE; static int display_attr = FALSE; +static int display_free_sections = FALSE; +static int display_summary = FALSE; /* a structure for handling the order command-line parameters come in */ struct handler_t { @@ -111,7 +123,7 @@ struct handler_t { }; -static const char *s_opts ="ADdFfhGgTO:V"; +static const char *s_opts ="ADdFfhGgsSTO:V"; static struct long_options l_opts[] = { {"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, @@ -179,6 +191,20 @@ static struct long_options l_opts[] = { { "att", no_arg, 'A' }, { "at", no_arg, 'A' }, { "a", no_arg, 'A' }, + { "freespace", no_arg, 's' }, + { "freespac", no_arg, 's' }, + { "freespa", no_arg, 's' }, + { "freesp", no_arg, 's' }, + { "frees", no_arg, 's' }, + { "free", no_arg, 's' }, + { "fre", no_arg, 's' }, + { "fr", no_arg, 's' }, + { "summary", no_arg, 'S' }, + { "summar", no_arg, 'S' }, + { "summa", no_arg, 'S' }, + { "summ", no_arg, 'S' }, + { "sum", no_arg, 'S' }, + { "su", no_arg, 'S' }, { NULL, 0, '\0' } }; @@ -206,7 +232,8 @@ static void usage(const char *prog) fprintf(stdout, " -D, --dsetmetadata Print dataset metadata\n"); fprintf(stdout, " -T, --dtypemetadata Print datatype metadata\n"); fprintf(stdout, " -A, --attribute Print attribute information\n"); - fprintf(stdout, "\n"); + fprintf(stdout, " -s, --freespace Print free-space information\n"); + fprintf(stdout, " -S, --Summary Print summary of storage information\n"); } @@ -415,6 +442,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) /* Update dataset metadata info */ iter->datasets_index_storage_size += oi->meta_size.obj.index_size; + iter->datasets_heap_storage_size += oi->meta_size.obj.heap_size; /* Update attribute metadata info */ ret = attribute_stats(iter, oi); @@ -422,7 +450,32 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) /* Get storage info */ storage = H5Dget_storage_size(did); - iter->dset_storage_size += storage; + + /* Gather layout statistics */ + dcpl = H5Dget_create_plist(did); + assert(dcpl > 0); + + lout = H5Pget_layout(dcpl); + assert(lout >= 0); + + /* Object header's total size for H5D_COMPACT layout includes raw data size */ + /* "storage" also includes H5D_COMPACT raw data size */ + if(lout == H5D_COMPACT) + iter->dset_ohdr_info.total_size -= storage; + + /* Track the layout type for dataset */ + (iter->dset_layouts[lout])++; + + /* Get the number of external files for the dataset */ + num_ext = H5Pget_external_count(dcpl); + assert (num_ext >= 0); + + /* Accumulate raw data size accordingly */ + if(num_ext) { + iter->nexternal += (unsigned long)num_ext; + iter->dset_external_storage_size += (unsigned long)storage; + } else + iter->dset_storage_size += storage; /* Gather dataspace statistics */ sid = H5Dget_space(did); @@ -505,22 +558,6 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) ret = H5Tclose(tid); assert(ret >= 0); - /* Gather layout statistics */ - dcpl = H5Dget_create_plist(did); - assert(dcpl > 0); - - lout = H5Pget_layout(dcpl); - assert(lout >= 0); - - /* Track the layout type for dataset */ - (iter->dset_layouts[lout])++; - - num_ext = H5Pget_external_count(dcpl); - assert (num_ext >= 0); - - if(num_ext) - iter->nexternal += (unsigned long)num_ext; - /* Track different filters */ if((nfltr = H5Pget_nfilters(dcpl)) >= 0) { if(nfltr == 0) @@ -653,6 +690,66 @@ lnk_stats(const char UNUSED *path, const H5L_info_t *li, void *_iter) return 0; } /* end lnk_stats() */ +/*------------------------------------------------------------------------- + * Function: freespace_stats + * + * Purpose: Gather statistics for free space sections in the file + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Vailin Choi; July 7th, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +freespace_stats(hid_t fid, iter_t *iter) +{ + H5F_sect_info_t *sect_info = NULL; /* Free space sections */ + ssize_t nsects; /* Number of free space sections */ + size_t u; /* Local index variable */ + + /* Query section information */ + if((nsects = H5Fget_free_sections(fid, H5FD_MEM_DEFAULT, 0, NULL)) < 0) + return(FAIL); + else if(nsects) { + if(NULL == (sect_info = (H5F_sect_info_t *)calloc((size_t)nsects, sizeof(H5F_sect_info_t)))) + return(FAIL); + nsects = H5Fget_free_sections(fid, H5FD_MEM_DEFAULT, (size_t)nsects, sect_info); + assert(nsects); + } /* end else-if */ + + for(u = 0; u < (size_t)nsects; u++) { + unsigned bin; /* "bin" the number of objects falls in */ + + if(sect_info[u].size < SIZE_SMALL_SECTS) + (iter->num_small_sects[(size_t)sect_info[u].size])++; + + /* Add section size to proper bin */ + bin = ceil_log10((unsigned long)sect_info[u].size); + if(bin >= iter->sect_nbins) { + /* Allocate more storage for section info */ + iter->sect_bins = (unsigned long *)realloc(iter->sect_bins, (bin + 1) * sizeof(unsigned long)); + assert(iter->sect_bins); + + /* Initialize counts for intermediate bins */ + while(iter->sect_nbins < bin) + iter->sect_bins[iter->sect_nbins++] = 0; + iter->sect_nbins++; + + /* Initialize count for this bin */ + iter->sect_bins[bin] = 1; + } /* end if */ + else + (iter->sect_bins[bin])++; + } /* end for */ + + if(sect_info) + free(sect_info); + + return 0; +} /* end freespace_stats() */ + /*------------------------------------------------------------------------- * Function: parse_command_line @@ -718,6 +815,16 @@ parse_command_line(int argc, const char *argv[]) display_dset = TRUE; break; + case 's': + display_all = FALSE; + display_free_sections = TRUE; + break; + + case 'S': + display_all = FALSE; + display_summary = TRUE; + break; + case 'h': usage(progname); leave(EXIT_SUCCESS); @@ -829,15 +936,20 @@ print_file_info(const iter_t *iter) static herr_t print_file_metadata(const iter_t *iter) { - printf("Object header size: (total/unused)\n"); - HDfprintf(stdout, "\tGroups: %Hu/%Hu\n", iter->group_ohdr_info.total_size, - iter->group_ohdr_info.free_size); - HDfprintf(stdout, "\tDatasets: %Hu/%Hu\n", iter->dset_ohdr_info.total_size, - iter->dset_ohdr_info.free_size); - HDfprintf(stdout, "\tDatatypes: %Hu/%Hu\n", iter->dtype_ohdr_info.total_size, - iter->dtype_ohdr_info.free_size); - - printf("Storage information:\n"); + printf("Storage information (in bytes):\n"); + HDfprintf(stdout, "\tSuperblock: %Hu\n", iter->super_size); + HDfprintf(stdout, "\tSuperblock extension: %Hu\n", iter->super_ext_size); + HDfprintf(stdout, "\tUser block: %Hu\n", iter->ublk_size); + + HDfprintf(stdout, "\tObject headers: (total/unused)\n"); + HDfprintf(stdout, "\t\tGroups: %Hu/%Hu\n", iter->group_ohdr_info.total_size, + iter->group_ohdr_info.free_size); + HDfprintf(stdout, "\t\tDatasets(exclude compact data): %Hu/%Hu\n", + iter->dset_ohdr_info.total_size, + iter->dset_ohdr_info.free_size); + HDfprintf(stdout, "\t\tDatatypes: %Hu/%Hu\n", iter->dtype_ohdr_info.total_size, + iter->dtype_ohdr_info.free_size); + HDfprintf(stdout, "\tGroups:\n"); HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->groups_btree_storage_size); HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->groups_heap_storage_size); @@ -847,14 +959,19 @@ print_file_metadata(const iter_t *iter) HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->attrs_heap_storage_size); HDfprintf(stdout, "\tChunked datasets:\n"); - HDfprintf(stdout, "\t\tB-tree: %Hu\n", iter->datasets_index_storage_size); + HDfprintf(stdout, "\t\tIndex: %Hu\n", iter->datasets_index_storage_size); + + HDfprintf(stdout, "\tDatasets:\n"); + HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->datasets_heap_storage_size); HDfprintf(stdout, "\tShared Messages:\n"); HDfprintf(stdout, "\t\tHeader: %Hu\n", iter->SM_hdr_storage_size); HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->SM_index_storage_size); HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->SM_heap_storage_size); - HDfprintf(stdout, "\tSuperblock extension: %Hu\n", iter->super_ext_size); + HDfprintf(stdout, "\tFree-space managers:\n"); + HDfprintf(stdout, "\t\tHeader: %Hu\n", iter->free_hdr); + HDfprintf(stdout, "\t\tAmount of free space: %Hu\n", iter->free_space); return 0; } @@ -1035,6 +1152,7 @@ print_dataset_info(const iter_t *iter) printf("Dataset storage information:\n"); HDfprintf(stdout, "\tTotal raw data size: %Hu\n", iter->dset_storage_size); + HDfprintf(stdout, "\tTotal external raw data size: %Hu\n", iter->dset_external_storage_size); printf("Dataset layout information:\n"); for(u = 0; u < H5D_NLAYOUTS; u++) @@ -1075,6 +1193,116 @@ print_dataset_info(const iter_t *iter) /*------------------------------------------------------------------------- + * Function: print_freespace_info + * + * Purpose: Prints information about free space in the file + * + * Return: Success: 0 + * + * Failure: Never fails + * + * Programmer: Vailin Choi; July 7th, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +print_freespace_info(const iter_t *iter) +{ + unsigned long power; /* Temporary "power" for bins */ + unsigned long total; /* Total count for various statistics */ + unsigned u; /* Local index variable */ + + printf("Small size free-space sections (< %u bytes):\n", (unsigned)SIZE_SMALL_SECTS); + total = 0; + for(u = 0; u < SIZE_SMALL_SECTS; u++) { + if(iter->num_small_sects[u] > 0) { + printf("\t# of sections of size %u: %lu\n", u, iter->num_small_sects[u]); + total += iter->num_small_sects[u]; + } /* end if */ + } /* end for */ + printf("\tTotal # of small size sections: %lu\n", total); + + printf("Free-space section bins:\n"); + + total = 0; + power = 1; + for(u = 1; u < iter->sect_nbins; u++) { + if(iter->sect_bins[u] > 0) { + printf("\t# of sections of size %lu - %lu: %lu\n", power, (power * 10) - 1, + iter->sect_bins[u]); + total += iter->sect_bins[u]; + } /* end if */ + power *= 10; + } /* end for */ + printf("\tTotal # of sections: %lu\n", total); + + return 0; +} /* print_freespace_info() */ + +/*------------------------------------------------------------------------- + * Function: print_storage_summary + * + * Purpose: Prints summary of the storage information in the file + * + * Return: Success: 0 + * + * Failure: Never fails + * + * Programmer: Vailin Choi; August 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +print_storage_summary(const iter_t *iter) +{ + hsize_t total_meta = 0; + hsize_t unaccount = 0; + float percent = 0.0; + + printf("Summary of storage information:\n"); + total_meta = + iter->super_size + iter->super_ext_size + iter->ublk_size + + iter->group_ohdr_info.total_size + + iter->dset_ohdr_info.total_size + + iter->dtype_ohdr_info.total_size + + iter->groups_btree_storage_size + + iter->groups_heap_storage_size + + iter->attrs_btree_storage_size + + iter->attrs_heap_storage_size + + iter->datasets_index_storage_size + + iter->datasets_heap_storage_size + + iter->SM_hdr_storage_size + + iter->SM_index_storage_size + + iter->SM_heap_storage_size + + iter->free_hdr; + + HDfprintf(stdout, " File metadata: %Hu bytes\n", total_meta); + HDfprintf(stdout, " Raw data: %Hu bytes\n", iter->dset_storage_size); + + percent = ((float)iter->free_space / (float)iter->filesize) * 100; + HDfprintf(stdout, " Amount/Percent of tracked free space: %Hu bytes/%3.1f%\n", + iter->free_space, percent); + + if(iter->filesize < (total_meta+iter->dset_storage_size+iter->free_space)) { + unaccount = (total_meta + iter->dset_storage_size + iter->free_space) - iter->filesize; + HDfprintf(stdout, " ??? File has %Hu more bytes accounted for that its size! ???\n", unaccount); + } + else { + unaccount = iter->filesize - (total_meta + iter->dset_storage_size + iter->free_space); + HDfprintf(stdout, " Unaccounted space: %Hu bytes\n", unaccount); + } + + HDfprintf(stdout, "Total space: %Hu bytes\n", + total_meta+iter->dset_storage_size+iter->free_space+unaccount); + + if(iter->nexternal) + HDfprintf(stdout, "External raw data: %Hu bytes\n", iter->dset_external_storage_size); + + return 0; +} /* print_storage_summary() */ + + +/*------------------------------------------------------------------------- * Function: print_file_statistics * * Purpose: Prints file statistics @@ -1098,6 +1326,7 @@ print_file_statistics(const iter_t *iter) display_dset = TRUE; display_dtype_metadata = TRUE; display_attr = TRUE; + display_free_sections = TRUE; } if(display_file) print_file_info(iter); @@ -1105,6 +1334,8 @@ print_file_statistics(const iter_t *iter) if(display_group) print_group_info(iter); if(display_dset) print_dataset_info(iter); if(display_attr) print_attr_info(iter); + if(display_free_sections) print_freespace_info(iter); + if(display_summary) print_storage_summary(iter); } @@ -1163,8 +1394,9 @@ main(int argc, const char *argv[]) iter_t iter; const char *fname = NULL; hid_t fid; + hid_t fcpl; struct handler_t *hand; - H5F_info_t finfo; + H5F_info2_t finfo; /* Disable error reporting */ H5Eset_auto2(H5E_DEFAULT, NULL, NULL); @@ -1190,16 +1422,33 @@ main(int argc, const char *argv[]) /* Initialize iter structure */ iter_init(&iter, fid); - /* Get storge info for SOHM's btree/list/heap and superblock extension */ - if(H5Fget_info(fid, &finfo) < 0) - warn_msg(progname, "Unable to retrieve SOHM info\n"); + if(H5Fget_filesize(fid, &iter.filesize) < 0) + warn_msg(progname, "Unable to retrieve file size\n"); + assert(iter.filesize >= 0); + + /* Get storge info for file-level structures */ + if(H5Fget_info2(fid, &finfo) < 0) + warn_msg(progname, "Unable to retrieve file info\n"); else { - iter.super_ext_size = finfo.super_ext_size; + iter.super_size = finfo.super.super_size; + iter.super_ext_size = finfo.super.super_ext_size; iter.SM_hdr_storage_size = finfo.sohm.hdr_size; iter.SM_index_storage_size = finfo.sohm.msgs_info.index_size; iter.SM_heap_storage_size = finfo.sohm.msgs_info.heap_size; + iter.free_space = finfo.free.tot_space; + iter.free_hdr = finfo.free.meta_size; } /* end else */ + if((fcpl = H5Fget_create_plist(fid)) < 0) + warn_msg(progname, "Unable to retrieve file creation property\n"); + + if(H5Pget_userblock(fcpl, &iter.ublk_size) < 0) + warn_msg(progname, "Unable to retrieve userblock size\n"); + + /* get information for free-space sections */ + if(freespace_stats(fid, &iter) < 0) + warn_msg(progname, "Unable to retrieve freespace info\n"); + /* Walk the objects or all file */ if(display_object) { unsigned u; diff --git a/tools/h5stat/h5stat_gentest.c b/tools/h5stat/h5stat_gentest.c index 3f933f2..e0ad3e2 100644 --- a/tools/h5stat/h5stat_gentest.c +++ b/tools/h5stat/h5stat_gentest.c @@ -40,18 +40,30 @@ */ static void gen_file(void) { - int ret, i; - hid_t fapl, gid; - hid_t file, type_id, space_id, attr_id, dset_id; - char name[30]; - char attrname[30]; + hid_t fcpl; /* File creation property */ + hid_t fapl; /* File access property */ + hid_t file; /* File id */ + hid_t gid; /* Group id */ + hid_t type_id; /* Datatype id */ + hid_t space_id; /* Dataspace id */ + hid_t attr_id; /* Attribute id */ + hid_t dset_id; /* Dataset id */ + char name[30]; /* Group name */ + char attrname[30]; /* Attribute name */ + int ret; /* Return value */ + int i; /* Local index variable */ fapl = H5Pcreate(H5P_FILE_ACCESS); ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); assert(ret >= 0); + /* Set file space handling strategy */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, 0); + assert(ret >= 0); + /* Create dataset */ - file = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + file = H5Fcreate(FILE, H5F_ACC_TRUNC, fcpl, fapl); for(i = 1; i <= NUM_GRPS; i++) { sprintf(name, "%s%d", GROUP_NAME,i); gid = H5Gcreate2(file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); diff --git a/tools/h5stat/testfiles/h5stat_filters-F.ddl b/tools/h5stat/testfiles/h5stat_filters-F.ddl index 6bc6cc2..78c7c5a 100644 --- a/tools/h5stat/testfiles/h5stat_filters-F.ddl +++ b/tools/h5stat/testfiles/h5stat_filters-F.ddl @@ -2,11 +2,14 @@ Expected output for 'h5stat -F h5stat_filters.h5' ############################# Filename: h5stat_filters.h5 -Object header size: (total/unused) - Groups: 48/8 - Datasets: 4936/1344 - Datatypes: 80/0 -Storage information: +Storage information (in bytes): + Superblock: 96 + Superblock extension: 0 + User block: 0 + Object headers: (total/unused) + Groups: 48/8 + Datasets(exclude compact data): 4136/1344 + Datatypes: 80/0 Groups: B-tree/List: 1200 Heap: 288 @@ -14,9 +17,13 @@ Storage information: B-tree/List: 0 Heap: 0 Chunked datasets: - B-tree: 31392 + Index: 31392 + Datasets: + Heap: 72 Shared Messages: Header: 0 B-tree/List: 0 Heap: 0 - Superblock extension: 0 + Free-space managers: + Header: 0 + Amount of free space: 0 diff --git a/tools/h5stat/testfiles/h5stat_filters-d.ddl b/tools/h5stat/testfiles/h5stat_filters-d.ddl index 42d0de3..3cc9071 100644 --- a/tools/h5stat/testfiles/h5stat_filters-d.ddl +++ b/tools/h5stat/testfiles/h5stat_filters-d.ddl @@ -15,7 +15,8 @@ Dataset dimension information: # of datasets of size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 9059 + Total raw data size: 8659 + Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 Dataset layout counts[CONTIG]: 2 diff --git a/tools/h5stat/testfiles/h5stat_filters-dT.ddl b/tools/h5stat/testfiles/h5stat_filters-dT.ddl index 2edcf7e..ae9121d 100644 --- a/tools/h5stat/testfiles/h5stat_filters-dT.ddl +++ b/tools/h5stat/testfiles/h5stat_filters-dT.ddl @@ -15,7 +15,8 @@ Dataset dimension information: # of datasets of size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 9059 + Total raw data size: 8659 + Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 Dataset layout counts[CONTIG]: 2 diff --git a/tools/h5stat/testfiles/h5stat_filters.ddl b/tools/h5stat/testfiles/h5stat_filters.ddl index f9adebd..6ce04fe 100644 --- a/tools/h5stat/testfiles/h5stat_filters.ddl +++ b/tools/h5stat/testfiles/h5stat_filters.ddl @@ -10,11 +10,14 @@ File information # of unique other: 0 Max. # of links to object: 1 Max. # of objects in group: 16 -Object header size: (total/unused) - Groups: 48/8 - Datasets: 4936/1344 - Datatypes: 80/0 -Storage information: +Storage information (in bytes): + Superblock: 96 + Superblock extension: 0 + User block: 0 + Object headers: (total/unused) + Groups: 48/8 + Datasets(exclude compact data): 4136/1344 + Datatypes: 80/0 Groups: B-tree/List: 1200 Heap: 288 @@ -22,12 +25,16 @@ Storage information: B-tree/List: 0 Heap: 0 Chunked datasets: - B-tree: 31392 + Index: 31392 + Datasets: + Heap: 72 Shared Messages: Header: 0 B-tree/List: 0 Heap: 0 - Superblock extension: 0 + Free-space managers: + Header: 0 + Amount of free space: 0 Small groups: Total # of small groups: 0 Group bins: @@ -46,7 +53,8 @@ Dataset dimension information: # of datasets of size 100 - 999: 1 Total # of datasets: 1 Dataset storage information: - Total raw data size: 9059 + Total raw data size: 8659 + Total external raw data size: 400 Dataset layout information: Dataset layout counts[COMPACT]: 1 Dataset layout counts[CONTIG]: 2 @@ -76,3 +84,7 @@ Small # of attributes: Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 diff --git a/tools/h5stat/testfiles/h5stat_help1.ddl b/tools/h5stat/testfiles/h5stat_help1.ddl index 0a86b4c..5808c49 100644 --- a/tools/h5stat/testfiles/h5stat_help1.ddl +++ b/tools/h5stat/testfiles/h5stat_help1.ddl @@ -14,4 +14,5 @@ Usage: h5stat [OPTIONS] file -D, --dsetmetadata Print dataset metadata -T, --dtypemetadata Print datatype metadata -A, --attribute Print attribute information - + -s, --freespace Print free-space information + -S, --Summary Print summary of storage information diff --git a/tools/h5stat/testfiles/h5stat_help2.ddl b/tools/h5stat/testfiles/h5stat_help2.ddl index e41eb9e..169787f 100644 --- a/tools/h5stat/testfiles/h5stat_help2.ddl +++ b/tools/h5stat/testfiles/h5stat_help2.ddl @@ -14,4 +14,5 @@ Usage: h5stat [OPTIONS] file -D, --dsetmetadata Print dataset metadata -T, --dtypemetadata Print datatype metadata -A, --attribute Print attribute information - + -s, --freespace Print free-space information + -S, --Summary Print summary of storage information diff --git a/tools/h5stat/testfiles/h5stat_newgrat.ddl b/tools/h5stat/testfiles/h5stat_newgrat.ddl index 4f9bcac..877e55a 100644 --- a/tools/h5stat/testfiles/h5stat_newgrat.ddl +++ b/tools/h5stat/testfiles/h5stat_newgrat.ddl @@ -10,24 +10,31 @@ File information # of unique other: 0 Max. # of links to object: 1 Max. # of objects in group: 35001 -Object header size: (total/unused) - Groups: 5145147/3220092 - Datasets: 414/312 - Datatypes: 0/0 -Storage information: +Storage information (in bytes): + Superblock: 48 + Superblock extension: 119 + User block: 0 + Object headers: (total/unused) + Groups: 5145147/3220092 + Datasets(exclude compact data): 414/312 + Datatypes: 0/0 Groups: B-tree/List: 470054 - Heap: 739102 + Heap: 739045 Attributes: B-tree/List: 2598 - Heap: 4442 + Heap: 4431 Chunked datasets: - B-tree: 0 + Index: 0 + Datasets: + Heap: 0 Shared Messages: Header: 0 B-tree/List: 0 Heap: 0 - Superblock extension: 0 + Free-space managers: + Header: 1350 + Amount of free space: 4463 Small groups: # of groups of size 0: 35000 Total # of small groups: 35000 @@ -45,6 +52,7 @@ Dataset dimension information: Total small datasets: 0 Dataset storage information: Total raw data size: 0 + Total external raw data size: 0 Dataset layout information: Dataset layout counts[COMPACT]: 0 Dataset layout counts[CONTIG]: 1 @@ -72,3 +80,15 @@ Attribute bins: # of objects with 100 - 999 attributes: 1 Total # of objects with attributes: 1 Max. # of attributes to objects: 100 +Small size free-space sections (< 10 bytes): + # of sections of size 1: 1 + # of sections of size 2: 12 + # of sections of size 3: 3 + # of sections of size 7: 1 + # of sections of size 8: 2 + Total # of small size sections: 19 +Free-space section bins: + # of sections of size 1 - 9: 19 + # of sections of size 10 - 99: 44 + # of sections of size 100 - 999: 18 + Total # of sections: 81 diff --git a/tools/h5stat/testfiles/h5stat_newgrat.h5 b/tools/h5stat/testfiles/h5stat_newgrat.h5 Binary files differindex 8fa406b..1db0205 100644 --- a/tools/h5stat/testfiles/h5stat_newgrat.h5 +++ b/tools/h5stat/testfiles/h5stat_newgrat.h5 diff --git a/tools/h5stat/testfiles/h5stat_tsohm.ddl b/tools/h5stat/testfiles/h5stat_tsohm.ddl index e536c33..a53e40c 100644 --- a/tools/h5stat/testfiles/h5stat_tsohm.ddl +++ b/tools/h5stat/testfiles/h5stat_tsohm.ddl @@ -10,11 +10,14 @@ File information # of unique other: 0 Max. # of links to object: 1 Max. # of objects in group: 3 -Object header size: (total/unused) - Groups: 51/2 - Datasets: 852/447 - Datatypes: 0/0 -Storage information: +Storage information (in bytes): + Superblock: 48 + Superblock extension: 40 + User block: 0 + Object headers: (total/unused) + Groups: 51/2 + Datasets(exclude compact data): 852/447 + Datatypes: 0/0 Groups: B-tree/List: 872 Heap: 120 @@ -22,12 +25,16 @@ Storage information: B-tree/List: 0 Heap: 0 Chunked datasets: - B-tree: 0 + Index: 0 + Datasets: + Heap: 0 Shared Messages: Header: 38 B-tree/List: 550 Heap: 1316 - Superblock extension: 40 + Free-space managers: + Header: 0 + Amount of free space: 0 Small groups: # of groups of size 3: 1 Total # of small groups: 1 @@ -44,6 +51,7 @@ Dataset dimension information: Total small datasets: 0 Dataset storage information: Total raw data size: 0 + Total external raw data size: 0 Dataset layout information: Dataset layout counts[COMPACT]: 0 Dataset layout counts[CONTIG]: 0 @@ -70,3 +78,7 @@ Small # of attributes: Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 diff --git a/tools/testfiles/file_space.ddl b/tools/testfiles/file_space.ddl new file mode 100644 index 0000000..37b3922 --- /dev/null +++ b/tools/testfiles/file_space.ddl @@ -0,0 +1,23 @@ +############################# +Expected output for 'h5dump -B file_space.h5' +############################# +HDF5 "file_space.h5" { +SUPER_BLOCK { + SUPERBLOCK_VERSION 2 + FREELIST_VERSION 0 + SYMBOLTABLE_VERSION 0 + OBJECTHEADER_VERSION 0 + OFFSET_SIZE 8 + LENGTH_SIZE 8 + BTREE_RANK 16 + BTREE_LEAF 4 + ISTORE_K 32 + FILE_SPACE_STRATEGY H5F_FILE_SPACE_AGGR_VFD + FREE_SPACE_THRESHOLD 10 +} +USER_BLOCK { + USERBLOCK_SIZE 0 +} +GROUP "/" { +} +} diff --git a/tools/testfiles/file_space.h5 b/tools/testfiles/file_space.h5 Binary files differnew file mode 100644 index 0000000..48184d3 --- /dev/null +++ b/tools/testfiles/file_space.h5 diff --git a/tools/testfiles/tboot1.ddl b/tools/testfiles/tboot1.ddl index 9a7cafc..8ffffc0 100644 --- a/tools/testfiles/tboot1.ddl +++ b/tools/testfiles/tboot1.ddl @@ -12,6 +12,8 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 + FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL + FREE_SPACE_THRESHOLD 1 } USER_BLOCK { USERBLOCK_SIZE 0 diff --git a/tools/testfiles/tboot2.ddl b/tools/testfiles/tboot2.ddl index d1484ca..51250fe 100644 --- a/tools/testfiles/tboot2.ddl +++ b/tools/testfiles/tboot2.ddl @@ -12,6 +12,8 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 + FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL + FREE_SPACE_THRESHOLD 1 } USER_BLOCK { USERBLOCK_SIZE 0 |