diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | examples/Makefile.am | 5 | ||||
| -rw-r--r-- | examples/Makefile.in | 5 | ||||
| -rw-r--r-- | examples/h5_shared_mesg.c | 325 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 1 |
5 files changed, 332 insertions, 5 deletions
@@ -137,6 +137,7 @@ ./examples/h5_mount.c ./examples/h5_reference.c ./examples/h5_ref2reg.c +./examples/h5_shared_mesg.c ./examples/ph5example.c diff --git a/examples/Makefile.am b/examples/Makefile.am index faf6940..b6f4b8d 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -30,14 +30,14 @@ endif # Use the boilerplate in config/examples.am instead. TEST_PROG = h5_write h5_read h5_extend_write h5_chunk_read h5_compound \ h5_group h5_select h5_attribute h5_mount h5_reference h5_drivers \ - h5_ref2reg h5_extlink h5_elink_unix2win + h5_ref2reg h5_extlink h5_elink_unix2win h5_shared_mesg # Install files # List all file that should be installed in examples directory INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c \ h5_compound.c h5_group.c h5_select.c h5_attribute.c h5_mount.c \ h5_reference.c h5_drivers.c h5_extlink.c h5_elink_unix2win.c \ - h5_ref2reg.c ph5example.c + h5_ref2reg.c h5_shared_mesg.c ph5example.c # Additional dependencies for each program are listed below. $(EXTRA_PROG): $(LIBHDF5) @@ -86,6 +86,7 @@ ph5example: $(srcdir)/ph5example.c h5_dtransform: $(srcdir)/h5_dtransform.c h5_extlink: $(srcdir)/h5_extlink.c $(EXTLINK_DIRS) h5_elink_unix2win: $(srcdir)/h5_elink_unix2win.c $(EXTLINK_DIRS) +h5_shared_mesg: $(srcdir)/h5_shared_mesg.c include $(top_srcdir)/config/examples.am include $(top_srcdir)/config/conclude.am diff --git a/examples/Makefile.in b/examples/Makefile.in index 68b1a26..ef7cdd9 100644 --- a/examples/Makefile.in +++ b/examples/Makefile.in @@ -314,7 +314,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog $(EXTLINK_DIRS) *.h5 # Use the boilerplate in config/examples.am instead. TEST_PROG = h5_write h5_read h5_extend_write h5_chunk_read h5_compound \ h5_group h5_select h5_attribute h5_mount h5_reference h5_drivers \ - h5_ref2reg h5_extlink h5_elink_unix2win + h5_ref2reg h5_extlink h5_elink_unix2win h5_shared_mesg # Install files @@ -322,7 +322,7 @@ TEST_PROG = h5_write h5_read h5_extend_write h5_chunk_read h5_compound \ INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c \ h5_compound.c h5_group.c h5_select.c h5_attribute.c h5_mount.c \ h5_reference.c h5_drivers.c h5_extlink.c h5_elink_unix2win.c \ - h5_ref2reg.c ph5example.c + h5_ref2reg.c h5_shared_mesg.c ph5example.c # The external link examples demonstrate how to use paths; they need @@ -568,6 +568,7 @@ ph5example: $(srcdir)/ph5example.c h5_dtransform: $(srcdir)/h5_dtransform.c h5_extlink: $(srcdir)/h5_extlink.c $(EXTLINK_DIRS) h5_elink_unix2win: $(srcdir)/h5_elink_unix2win.c $(EXTLINK_DIRS) +h5_shared_mesg: $(srcdir)/h5_shared_mesg.c # How to create EXAMPLEDIR if it doesn't already exist $(EXAMPLEDIR): diff --git a/examples/h5_shared_mesg.c b/examples/h5_shared_mesg.c new file mode 100644 index 0000000..f6e806c --- /dev/null +++ b/examples/h5_shared_mesg.c @@ -0,0 +1,325 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * This program illustrates the usage of HDF5's implicit message sharing + * feature, which can be used to save space when the same messages are + * used many times in a file. + * + * This example creates a standard file using file creation property lists + * to control which messages are shared. Messages that can be shared are + * datatypes, dataspaces, attributes, fill values, and filter pipelines. + * + */ + +#include <stdlib.h> + +#include "hdf5.h" + +#define NUM_DATASETS 40 +const char* DSETNAME[] = { + "dataset0", "dataset1", + "dataset2", "dataset3", + "dataset4", "dataset5", + "dataset6", "dataset7", + "dataset8", "dataset9", + "dataset10", "dataset11", + "dataset12", "dataset13", + "dataset14", "dataset15", + "dataset16", "dataset17", + "dataset18", "dataset19", + "dataset20", "dataset21", + "dataset22", "dataset23", + "dataset24", "dataset25", + "dataset26", "dataset27", + "dataset28", "dataset29", + "dataset30", "dataset31", + "dataset32", "dataset33", + "dataset34", "dataset35", + "dataset36", "dataset37", + "dataset38", "dataset39", + NULL +}; + +herr_t create_standard_file(const char *filename, hid_t fcpl); + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Enables shared messages using File Creation Property Lists + * and creates files using these settings. + * + *------------------------------------------------------------------------- + */ +int main(void) +{ + hid_t fcpl_id; + herr_t ret; + + /* Create a file creation property list */ + fcpl_id = H5Pcreate(H5P_FILE_CREATE); + if(fcpl_id < 0) goto error; + + /* The file creation property list is the default list right now. + * Create a file using it (this is the same as creating a file with + * H5P_DEFAULT). Implicit shared messages will be disabled. + */ + ret = create_standard_file("default_file.h5", fcpl_id); + if(ret < 0) goto error; + + /* There are five kinds of messages that can be shared: datatypes, + * dataspaces, attributes, fill values, and filter pipelines. + * Shared messages are stored in up to five "indexes," where each + * index can contain one or more types of message. Using more indexes + * will result in more overhead for sharing, but can also provide + * more "tunability" and may affect caching performance. + */ + /* To begin with, use only one index. */ + ret = H5Pset_shared_mesg_nindexes(fcpl_id, 1); + if(ret < 0) goto error; + + /* Each index has a "minimum message size" for a message of that + * type to be shared. Since sharing a message creates some overhead, + * this is to prevent this overhead for very small messages when little + * space would be saved by sharing them anyway. + * If the content of the file isn't known beforehand, it's probably best + * to set the minimum size "high"; over 100 or 200 bytes. If the content + * of the file is known, this value can be used to trade space saved for + * performance lost. The smaller this value is, the more messages will + * be shared, so the more overhead will be incurred. + * This value is in bytes. A shared message involves about 30 bytes of + * overhead. Note that even messages that are only written once will + * require this overhead (since they "might" be shared in the future), + * so setting the minimum size too low may result in a file actually growing + * in size. + * For this example case, we'll set the minimum sharing size to be small + * since we know that every message the "standard" file uses will be + * repeated many times. + */ + /* The other property that each index has is the kinds of messages that + * it holds. For the simple case, we'll put every message that could be + * shared in this single index. + */ + ret = H5Pset_shared_mesg_index(fcpl_id, 1, H5O_MESG_ALL_FLAG, 40); + if(ret < 0) goto error; + + /* The other property that can be set for shared messages is the + * list/B-tree cutoff for the indexes. + * Each shared message index beins life as a simple list of messages + * and becomes a B-tree when "too many" messages are written to it. + * This keeps the indexes simple when only a few messages are shared, + * but allows them to scale for many messages. If many messages are + * deleted from the B-tree, it scales back down into a list. + * A "resonable" setting for maximum list size and minimum btree size + * depends on what kinds of messages will be stored in the file. + * These numbers are the same for all indexes in a file. + * We'll guess at some numbers, though we could just as easily have kept + * the default values. The first value is the maximum list size, the + * second the minimum B-tree size. + */ + ret = H5Pset_shared_mesg_phase_change(fcpl_id, 30, 20); + if(ret < 0) goto error; + + /* Now create a file with this property list. After the FCPL is used, + * everything is automatic; messages will be shared and this will be + * completely transparent to the user. Even if the file is closed + * and re-opened, this settings will be saved and applied to messages + * written later. + */ + ret = create_standard_file("one_index_file.h5", fcpl_id); + if(ret < 0) goto error; + + /* Now try some variations on this. The FCPL hasn't been closed, so + * we don't need to re-create it. + * For instance, if we set the index to only share very large + * messages, none of the messages we write will qualify and the file + * will be about the same size as a normal file (with just a little extra + * overhead). + */ + ret = H5Pset_shared_mesg_index(fcpl_id, 1, H5O_MESG_ALL_FLAG, 1000); + if(ret < 0) goto error; + + ret = create_standard_file("only_huge_mesgs_file.h5", fcpl_id); + if(ret < 0) goto error; + + + /* Or, suppose we only wanted to shared dataspaces and + * attributes (which might make sense if we were going to use committed + * datatypes). We could change the flags on the index: + */ + ret = H5Pset_shared_mesg_index(fcpl_id, 1, H5O_MESG_SDSPACE_FLAG | H5O_MESG_ATTR_FLAG, 40); + if(ret < 0) goto error; + + ret = create_standard_file("only_dspaces_and_attrs_file.h5", fcpl_id); + if(ret < 0) goto error; + + + /* We create a second index and put attributes in it to separate them from + * datatypes and dataspaces (and then run some performance metrics to + * see whether this improved caching performance). + */ + ret = H5Pset_shared_mesg_nindexes(fcpl_id, 2); + if(ret < 0) goto error; + ret = H5Pset_shared_mesg_index(fcpl_id, 1, H5O_MESG_DTYPE_FLAG | H5O_MESG_SDSPACE_FLAG, 40); + if(ret < 0) goto error; + ret = H5Pset_shared_mesg_index(fcpl_id, 2, H5O_MESG_ATTR_FLAG, 40); + if(ret < 0) goto error; + + ret = create_standard_file("separate_indexes_file.h5", fcpl_id); + if(ret < 0) goto error; + + /* We can try twiddling the "phase change" values and see what it does to + * the file size. Since there's only a few different messages (two + * datatypes, two dataspaces, and one attribute), using smaller lists will + * save some space. + */ + ret = H5Pset_shared_mesg_nindexes(fcpl_id, 1); + if(ret < 0) goto error; + ret = H5Pset_shared_mesg_index(fcpl_id, 1, H5O_MESG_ALL_FLAG, 40); + if(ret < 0) goto error; + + ret = H5Pset_shared_mesg_phase_change(fcpl_id, 5, 0); + if(ret < 0) goto error; + + ret = create_standard_file("small_lists_file.h5", fcpl_id); + if(ret < 0) goto error; + + /* Or we could create indexes that are never lists, but are created as + * B-trees. We do this by setting the "maximum list size" to zero. + */ + ret = H5Pset_shared_mesg_phase_change(fcpl_id, 0, 0); + if(ret < 0) goto error; + + ret = create_standard_file("btrees_file.h5", fcpl_id); + if(ret < 0) goto error; + + + /* Obviously there are a lot more permutations of these options possible. + * Performance will often be a tradeoff of speed for space, but will + * depend a great deal on the specific application. If performance is + * important, the best thing to do is to play with these settings to find + * the ones that work best for you. + * Please let The HDF Group (help@hdfgroup.org) know what you find! + */ + + + /* Close the property list */ + ret = H5Pclose(fcpl_id); + if(ret < 0) goto error; + return 0; + +error: + return -1; +} + +/*------------------------------------------------------------------------- + * Function: create_standard_file + * + * Purpose: A helper functon for the example. Creates an HDF5 file + * with many repeated messages using the file creation + * property list FCPL. + * + * This function only uses datatypes, dataspaces, and + * attributes. Fill values and filter pipelines can also + * be shared in the same way (i.e., by enabling sharing in + * the FCPL and writing the same message more than once). + *------------------------------------------------------------------------- + */ +herr_t +create_standard_file(const char *filename, hid_t fcpl_id) +{ + hid_t file_id=-1; + hid_t type_id=-1, temp_type_id=-1; + hsize_t dims[] = {10, 9, 8, 7, 6, 5, 4, 3, 2, 1}; + hid_t space_id=-1; + hid_t attr_type_id = -1; + hid_t attr_space_id = -1; + int attr_data[] = {1,2,3,4,5,6,7,8,9,0}; + hid_t dset_id=-1; + hid_t attr_id=-1; + int x; + herr_t ret; + + /* Create the file */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); + if(file_id < 0) goto error; + + /* Create the datatype we'll be using. Generally, sharing messages + * is most useful when the message is complex and takes more space on + * disk, so this type will be an array type rather than an atomic type. + * However, any type can be shared. + */ + temp_type_id = H5Tarray_create(H5T_NATIVE_INT, 10, dims, NULL); + if(temp_type_id < 0) goto error; + type_id = H5Tarray_create(temp_type_id, 10, dims, NULL); + if(type_id < 0) goto error; + ret = H5Tclose(temp_type_id); + if(ret < 0) goto error; + + /* Create the dataspace we'll be using. + * Again, create a more complex dataspace so that more space will + * be saved when we share it. + */ + space_id = H5Screate_simple(10, dims, dims); + if(space_id < 0) goto error; + + /* Create a datatype and dataspace for the attributes we'll be creating. + * The datatype will be a single integer, and each attribute will hold + * 10 integers. + */ + attr_type_id = H5Tcopy(H5T_NATIVE_INT); + if(attr_type_id < 0) goto error; + attr_space_id = H5Screate_simple(1, dims, dims); + if(attr_space_id < 0) goto error; + + + /* Begin using the messages many times. Do this by creating datasets + * that use this datatype, dataspace, and have this attribute. + */ + for(x=0; x<NUM_DATASETS; ++x) { + /* Create a dataset */ + dset_id = H5Dcreate(file_id, DSETNAME[x], type_id, space_id, H5P_DEFAULT); + if(dset_id < 0) goto error; + + /* Create an attribute on the dataset */ + attr_id = H5Acreate(dset_id, "attr_name", attr_type_id, attr_space_id, H5P_DEFAULT); + if(attr_id < 0) goto error; + /* Write data to the attribute */ + ret = H5Awrite(attr_id, H5T_NATIVE_INT, attr_data); + if(ret < 0) goto error; + + ret = H5Aclose(attr_id); + if(ret < 0) goto error; + ret = H5Dclose(dset_id); + if(ret < 0) goto error; + } + + /* Close all open IDs */ + ret = H5Tclose(attr_type_id); + if(ret < 0) goto error; + ret = H5Sclose(attr_space_id); + if(ret < 0) goto error; + ret = H5Tclose(type_id); + if(ret < 0) goto error; + ret = H5Sclose(space_id); + if(ret < 0) goto error; + ret = H5Fclose(file_id); + if(ret < 0) goto error; + + return 0; + +error: + return -1; +} + diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 83ae330..515120b 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -236,7 +236,6 @@ H5_DLL herr_t H5Pset_sym_k(hid_t plist_id, unsigned ik, unsigned lk); H5_DLL herr_t H5Pget_sym_k(hid_t plist_id, unsigned *ik/*out*/, unsigned *lk/*out*/); H5_DLL herr_t H5Pset_istore_k(hid_t plist_id, unsigned ik); H5_DLL herr_t H5Pget_istore_k(hid_t plist_id, unsigned *ik/*out*/); -H5_DLL herr_t H5Pset_shared_mesgs(hid_t plist_id, unsigned nindexes, const unsigned mesg_type_flags[]); H5_DLL herr_t H5Pset_shared_mesg_nindexes(hid_t plist_id, unsigned nindexes); H5_DLL herr_t H5Pget_shared_mesg_nindexes(hid_t plist_id, unsigned *nindexes); H5_DLL herr_t H5Pset_shared_mesg_index(hid_t plist_id, unsigned index_num, unsigned mesg_type_flags, unsigned min_mesg_size); |