diff options
Diffstat (limited to 'doxygen')
25 files changed, 3068 insertions, 270 deletions
diff --git a/doxygen/CMakeLists.txt b/doxygen/CMakeLists.txt index ca36ef1..920fafa 100644 --- a/doxygen/CMakeLists.txt +++ b/doxygen/CMakeLists.txt @@ -27,6 +27,7 @@ if (DOXYGEN_FOUND) set (DOXYGEN_EXTERNAL_SEARCH NO) set (DOXYGEN_SEARCHENGINE_URL) set (DOXYGEN_STRIP_FROM_PATH ${HDF5_SOURCE_DIR}) + set (DOXYGEN_STRIP_FROM_INC_PATH ${HDF5_SOURCE_DIR}) set (DOXYGEN_PREDEFINED "H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD") # This configure and individual custom targets work together diff --git a/doxygen/Doxyfile.in b/doxygen/Doxyfile.in index d37222e..7958767 100644 --- a/doxygen/Doxyfile.in +++ b/doxygen/Doxyfile.in @@ -179,7 +179,7 @@ STRIP_FROM_PATH = @DOXYGEN_STRIP_FROM_PATH@ # specify the list of include paths that are normally passed to the compiler # using the -I flag. -STRIP_FROM_INC_PATH = +STRIP_FROM_INC_PATH = @DOXYGEN_STRIP_FROM_INC_PATH@ # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't diff --git a/doxygen/aliases b/doxygen/aliases index d5acbee..71a5488 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -297,7 +297,7 @@ ALIASES += ref_rfc20071111="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/AURA-co ALIASES += ref_rfc20071018="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_NaNsHDF5.pdf\"><tt>NaN</tt> detection in HDF5</a>" ALIASES += ref_rfc20070801="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Metadata_Journaling_RFC.pdf\">Metadata Journaling to Improve Crash Survivability</a>" ALIASES += ref_rfc20070413="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/API_Compatibility_RFC.txt.pdf\">API Compatibility Strategies for HDF5</a>" -ALIASES += ref_rfc20070115="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/PrivateHeap.pdf\">A 'Private' Heap for HDF5</a>" +ALIASES += ref_rfc20070115="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/PrivateHeap.pdf\">A \"Private\" Heap for HDF5</a>" ALIASES += ref_rfc20060623="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/coll_ind_dd6.pdf\">Performance Comparison of Collective I/O and Independent I/O with Derived Datatypes</a>" ALIASES += ref_rfc20060604="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/h5stat_spec_v3_2006-06-04.pdf\"><tt>h5stat</tt> tool</a>" ALIASES += ref_rfc20060505="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Simple%20Performance%20Test%20on%20Fletcher32%20Filter.pdf\">Simple Performance Test on Fletcher32 Filter</a>" diff --git a/doxygen/dox/About.dox b/doxygen/dox/About.dox index 32930a8..a8b31d7 100644 --- a/doxygen/dox/About.dox +++ b/doxygen/dox/About.dox @@ -12,12 +12,116 @@ of <em>documentation done right</em>. \section about_documentation Documentation about Documentation -\li \todo Describe how to add a reference or a new RFC -\li \todo Describe how to add an example -\li \todo Describe how to include plain HTML -\li \todo Describe how to add an API macro -\li \todo Describe the custom commands -\li \todo Describe the S3 bucket layout and update routine -\li \todo Link the RM template +In this section, we describe common documentation maintenance tasks. + +\subsection plain_html Including Plain HTML Pages + +The most common use case for this is the inclusion of older documentation. +New documentation should, whenever possible, be created using Doxygen markdown! + +Use Doxygen's <a href="https://www.doxygen.nl/manual/commands.html#cmdhtmlinclude"><code>htmlinclude</code></a> +special command to include existing plain HTML pages. + +An example from this documentation set can be seen +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/dox/FileFormatSpec.dox">here</a>. + +\subsection new_rm_entry Creating a New Reference Manual Entry + +Please refer to the \ref RMT for guidance on how to create a new reference manual entry. + +\subsubsection new_example Adding and Referencing API Examples + +For each HDF5 module, such as \Code{H5F}, there is an examples source file called +\Code{H5*_examples.c}. For example, the \Code{H5F} API examples are located in +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/examples/H5F_examples.c"> +<code>H5F_examples.c</code></a>. Examples are code blocks marked as Doxygen +<a href="https://www.doxygen.nl/manual/commands.html#cmdsnippet">snippets</a>. +For example, the source code for the H5Fcreate() API sample is located between +the +\verbatim +//! <!-- [create] --> +... +//! <!-- [create] --> +\endverbatim +comments in +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/examples/H5F_examples.c"> +<code>H5F_examples.c</code></a>. + +Add a new API example by adding a new code block enclosed between matching +snippet tags. The name of the tag is usually the function name stripped of +the module prefix. + +The inclusion of such a block of code can then be triggered via Doxygen's +<a href="https://www.doxygen.nl/manual/commands.html#cmdsnippet"><code>snippet</code></a> +special command. For example, the following markup +\verbatim +* \snippet H5F_examples.c create +\endverbatim +yields +\snippet H5F_examples.c create + +\subsubsection api_macro Adding an API Macro + +API macros are handled by the <code>api_vers_2, api_vers_3, api_vers_4</code> +custom commands. The numbers indicate the number of potential API function +mappings. For example, H5Acreate() has two potential mappings, H5Acreate1() and +H5Acreate2(). To trigger the creation of a reference manual entry for H5Acreate() +use the following markup: +\verbatim +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} +\endverbatim +This yields: + +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} + +\subsection custom_commands Creating Custom Commands + +See Doxygen's <a href="https://www.doxygen.nl/manual/custcmd.html">Custom Commands documentation</a> +as a general reference. + +All custom commands for this project are located in the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file in the <a href="https://github.com/HDFGroup/hdf5/tree/develop/doxygen"><tt>doxygen</tt></a> +subdirectory of the <a href="https://github.com/HDFGroup/hdf5">main HDF5 repo</a>. + +The custom commands are grouped in sections. Find a suitable section for your command or +ask for help if unsure! + +\subsection new_rfc Adding a New RFC or Referencing an Existing RFC + +For ease of reference, we define custom commands for each RFC in the <tt>RFCs</tt> section +of the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file. For example the custom command \Code{ref_rfc20141210} can be used to insert a +reference to "RFC: Virtual Object Layer". In other words, the markup +\verbatim +\ref_rfc20141210 +\endverbatim +yields a clickable link: + +\ref_rfc20141210 + +To add a new RFC, add a custom command for the RFC to the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file. The naming convention for the custom command is \Code{ref_rfcYYYYMMDD}, +where \Code{YYYYMMDD} is the ID of the RFC. The URL is composed of the prefix +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/ +\endverbatim +and the name of your RFC file, typically, a PDF file, i.e., the full URL would +be +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/my_great_rfc_name.pdf +\endverbatim + +\subsection hosting How Do Updates and Changes Get Published? + +Currently, the files underlying this documentation website are stored in an +bucket on AWS S3. The top-level bucket is <pre>s3://docs.hdfgroup.org/hdf5/</pre> +There are folders for the <tt>development</tt> branch and all supported release +version. + +Talk to your friendly IT-team if you need write access, or you need someone to +push an updated version for you! */
\ No newline at end of file diff --git a/doxygen/dox/FTS.dox b/doxygen/dox/FTS.dox new file mode 100644 index 0000000..9dae7c1 --- /dev/null +++ b/doxygen/dox/FTS.dox @@ -0,0 +1,8 @@ +/** \page FTS Full-Text Search + +\htmlonly +<script async src="https://cse.google.com/cse.js?cx=75c754173f9b90804"></script> +<div class="gcse-search"></div> +\endhtmlonly + +*/
\ No newline at end of file diff --git a/doxygen/dox/Glossary.dox b/doxygen/dox/Glossary.dox new file mode 100644 index 0000000..9ccd27d --- /dev/null +++ b/doxygen/dox/Glossary.dox @@ -0,0 +1,565 @@ +/** \page GLS Glossary + +\section GLS_A A + +<DL> + <DT>Array datatype</DT> + <DD>A family of HDF5 datatypes whose elements are arrays of a fixed rank (≤ + 32) and fixed finite extent. All array elements must be of the same HDF5 + datatype.</DD> +</DL> + +<DL> + <DT>Array variable</DT> + <DD><P>A variable that can store (logically) dense, rectilinear, multidimensional + arrays of elements of a given HDF5 datatype.</P> + <P>The combination of array rank (dimensionality) and extent is called an + array variable's shape. This includes the degenerate array shapes of a + singleton (scalar) and the empty array (null).</P> + <P>The array element datatype is sometimes referred to as the array + variable's type, which is not entirely accurate because the array variable's + type is 'array of element type' rather than 'element type'.</P> + <P>In HDF5, there are two kinds of array variables, attributes and datasets, + and the distinction is functional (i.e., how they can be used) rather than + conceptual. Attributes are commonly used for descriptive "light-weight" + HDF5 object metadata while datasets are HDF5 objects used to store + "heavy-weight" problem-sized data.</P> + </DD> +</DL> + +<DL> + <DT>Attribute</DT> + <DD><P>A named array variable that is associated with an HDF5 object, its + owner or attributee, and used to represent application domain-specific + metadata of the object. Intuitively, the set of an object's attributes can + be thought of as its key-value pair collection. Attribute names (keys) can + be arbitrary Unicode strings, but must be unique per object, i.e., an + object can have at most one attribute with a given name.</P> + <P>A scalar attribute is an attribute backed by a singleton array + variable. A null attribute is attribute backed by an empty array + variable.</P> +</DD> +</DL> + +\section GLS_B B + +<DL> + <DT>Bitfield datatype</DT> + <DD>A family of HDF5 datatypes whose elements are fixed-width bit fields.</DD> +</DL> + +\section GLS_C C + +<DL> + <DT>Chunked layout</DT> + <DD> + <P>A dataset storage layout where the dataset elements are partitioned into + fixed-size multidimensional chunks or tiles. Chunked layout is mandatory + for datasets with one or more dimensions of indefinite (infinite) extent + or where compression or other filters are applied to the dataset elements.</P> + <P>Chunked layout may improve I/O performance for certain access patterns.</P> +</DD> +</DL> + +<DL> + <DT>Committed datatype</DT> + <DD>An immutable kind of HDF5 object that is used to store an HDF5 datatype + definition, which can be referenced by multiple array variables. When linked + to an HDF5 group, a committed datatype can be located by an HDF5 path name, + and is sometimes called a named datatype.</DD> +</DL> + +<DL> + <DT>Compact layout</DT> + <DD></DD> +</DL> + +<DL> + <DT>Compound datatype</DT> + <DD> + <P>A family of HDF5 datatypes whose elements are records with named fields + of other HDF5 datatypes. Currently, on ASCII field names are supported.</P> + <P>Similar to a <CODE>struct</CODE> in C or a <CODE>COMMON</CODE> block in + Fortran.</P> +</DD> +</DL> + +<DL> + <DT>Contiguous layout</DT> + <DD>A dataset storage layout where the dataset elements are physically stored + in an HDF5 file as a contiguous block of bytes.</DD> +</DL> + +\section GLS_D D + +<DL> + <DT>Dataset</DT> + <DD> + <P>A kind of HDF5 object, a linked array variable. which can be located in + an HDF5 file through a path name. Datasets are commonly used to store + "heavy-weight" problem-sized data.</P> + <P>The HDF5 library offers a lot of features aimed at optimized dataset + access and storage, including compression and partial I/O.</P> +</DD> +</DL> + +<DL> + <DT>Dataspace</DT> + <DD>The shape of an array variable. With the exception of degenerate cases + (empty set, singleton), this is a rectilinear lattice or grid of a certain + rank (dimensionality) and extent.</DD> +</DL> + +<DL> + <DT>Datatype</DT> + <DD> + <P>An HDF5 datatype consists of an abstract data type (a set of elements) + and a bit-level representation of these elements in storage such as an HDF5 + file or memory.</P> + <P>The HDF5 library comes with a large set of predefined datatypes and + offers mechanisms for creating user-defined datatypes.</P> + <P>The ten major families or classes of HDF5 datatypes are:</P> + <UL> + <LI>Integer datatypes</LI> + <LI>Floating-point number datatypes</LI> + <LI>String datatypes</LI> + <LI>Bitfield datatypes</LI> + <LI>Opaque datatypes</LI> + <LI>Compound datatypes</LI> + <LI>Reference datatypes</LI> + <LI>Enumerated datatypes</LI> + <LI>Variable-length sequence datatypes</LI> + <LI>Array datatypes</LI> + </UL> +</DD> +</DL> + +\section GLS_E E + +<DL> + <DT>Enumeration datatype</DT> + <DD>A family of HDF5 datatypes whose elements represent named integer values + called members or enumerators. Currently, only ASCII names are supported.</DD> +</DL> + +<DL> + <DT>External layout</DT> + <DD>A form of contiguous layout where a dataset's elements are physically + stored in unformatted binary files outside the HDF5 file.</DD> +</DL> + +<DL> + <DT>External link</DT> + <DD>An HDF5 link whose destination is specified as a pair of an HDF5 file name +and an HDF5 path name in that file.</DD> +</DL> + +\section GLS_F F + +<DL> + <DT>Field</DT> + <DD>See compound datatype.</DD> +</DL> + +<DL> + <DT>File</DT> + <DD> + <OL> + <LI>A byte stream (in a storage context such as a file system or in + memory) formatted according to the HDF5 File Format Specification.</LI> + <LI>A (logical) container for HDF5 objects.</LI> + </OL> + </DD> +</DL> + +<DL> + <DT>File format</DT> + <DD></DD> +</DL> + +<DL> + <DT>Fill value</DT> + <DD></DD> +</DL> + +<DL> + <DT>Filter</DT> + <DD></DD> +</DL> + +\section GLS_G G + +<DL> + <DT>Group</DT> + <DD> + <P>A kind of HDF5 object that stores a collection of HDF5 links. Each HDF5 + file contains at least one group, it's root group.</P> + <P>Among the destinations of an HDF5 group's links may be other HDF5 groups + (including the group itself!). This ability is sometimes referred to as the + closure property of groups. It is the basis for creating hierarchical or + more general graph-like structures.</P> +</DD> +</DL> + +\section GLS_H H + +<DL> + <DT>Hard link</DT> + <DD>An HDF5 link whose destination is specified (internally) as the address of + an HDF5 object in the same HDF5 file.</DD> +</DL> + +<DL> + <DT>Hierarchy</DT> + <DD>See group.</DD> +</DL> + +<DL> + <DT>Hyperslab</DT> + <DD> + <P>A regular multidimensional pattern described by four vectors whose length + equals the rank of the pattern.</P> + <OL> + <LI><CODE>start</CODE> - the offset where the first block of the hyperslab begins</LI> + <LI><CODE>stride</CODE> - the offset between pattern blocks</LI> + <LI><CODE>count</CODE> - the number of blocks</LI> + <LI><CODE>block</CODE> - the extent of an individual pattern block</LI> + </OL> + <P>For example, the black squares on a (two-dimensional) chessboard with + origin at <CODE>(0,0)</CODE> can be represented as the union of two + hyperslabs representing the even <CODE>(0,2,4,6)</CODE> and + odd <CODE>(1,3,5,7)</CODE> rows, respectively.</P> + <IMG SRC="https://upload.wikimedia.org/wikipedia/commons/thumb/d/d7/Chessboard480.svg/176px-Chessboard480.svg.png"/> + <P>The hyperslab parameters for the even rows are: <CODE>start (0,0)</CODE>, + <CODE>stride (2,2)</CODE>, <CODE>count (4,4)</CODE>, <CODE>block + (1,1)</CODE>. Likewise the parameters for the odd rows are: <CODE>start + (1,1)</CODE>, <CODE>stride (2,2)</CODE>, <CODE>count + (4,4)</CODE>, <CODE>block (1,1)</CODE>.</P> +</DD> +</DL> + +\section GLS_I I + +<DL> + <DT>Identifier</DT> + <DD>An opaque, transient handle used by the HDF5 library to manipulate + in-memory representations of HDF5 items.</DD> +</DL> + +\section GLS_L L + +<DL> + <DT>Library</DT> + <DD></DD> +</DL> + +<DL> + <DT>Link</DT> + <DD> + <P>A named, uni-directional association between a source and a + destination. In HDF5, the source is always the HDF5 group that hosts the + link in its link collection.</P> + <P>There are several ways to specify a link's destination:</P> + <UL> + <LI>The address of an HDF5 object in the same HDF5 file; so-called hard + link.</LI> + <LI>A path name in the same or a different file; so-called soft or + external link.</LI> + <LI>User-defined</LI> + </UL> + <P>A link name can be any Unicode string that does not contain slashes + (<CODE>"/"</CODE>) or consists of a single dot character + (<CODE>"."</CODE>). A link name must be unique in a group's link + collection.</P> + </DD> +</DL> + +\section GLS_M M + +<DL> + <DT>Metadata</DT> + <DD>Data that in a given context has a descriptive or documentation function + for other data. Typically, the metadata is small compared to the data it + describes.</DD> +</DL> + +<DL> + <DT>Member</DT> + <DD> + <P>A link destination is sometimes referred to as a member of the link's + source (group). This way of speaking invites confusion: A destination (e.g., + object) can be the destination of multiple links in the same (!) or + different groups. It would then be a "member" of a given group with + multiplicity greater than one and be a member of multiple groups.</P> + <P> It is the link that is a member of the group's link collection and not + the link destination.</P> + </DD> +</DL> + +\section GLS_N N + +<DL> + <DT>Name</DT> + <DD> + <P>A Unicode string that depending on the item it names might be subject to + certain character restrictions, such as ASCII-encoded only. In HDF5, the + user might encounter the following names:</P> + <UL> + <LI>A link name</LI> + <LI>A path name</LI> + <LI>An attribute name</LI> + <LI>A field name (compound datatypes)</LI> + <LI>A constant name (enumeration datatypes)</LI> + <LI>A tag name (opaque datatypes)</LI> + <LI>A file name</LI> + </UL> + </DD> +</DL> + + +<DL> + <DT>Named datatype</DT> + <DD>See committed datatype.</DD> +</DL> + +<DL> + <DT>Null dataspace</DT> + <DD>A shape which represents the empty set. Array variables with this shape + cannot store any values.</DD> +</DL> + +\section GLS_O O + +<DL> + <DT>Object</DT> + <DD>An HDF5 group, dataset or named datatype; an HDF5 item that can be linked + to zero or more groups and decorated with zero or more HDF5 attributes.</DD> +</DL> + +<DL> + <DT>Object reference</DT> + <DD> + <OL> + <LI>A datatype for representing references to objects in a file.</LI> + <LI>A value of the object reference datatype.</LI> + </OL> + </DD> +</DL> + +<DL> + <DT>Opaque datatype</DT> + <DD>A family of HDF5 datatypes whose elements are byte sequences of a given + fixed length. An opaque datatype can be tagged with a sequence of up to 256 + ASCII characters, e.g., MIME code.</DD> +</DL> + +\section GLS_P P + +<DL> + <DT>Path name</DT> + <DD>A Unicode string that is the concatenation of link names separated by + slashes (<CODE>'/'</CODE>). In HDF5, path names are used to locate and refer + to HDF5 objects.</DD> +</DL> + +<DL> + <DT>Plugin</DT> + <DD>An HDF5 library feature or capability that can be added dynamically at + application run time rather than library compilation time. Plugins are + usually implemented as shared libraries, and their discovery and loading + behavior can be controlled programmatically or through environment + variables. + </DD> +</DL> + +<DL> + <DT>Point selection</DT> + <DD>A dataspace selection that consists of a set of points (coordinates) in + the same dataspace.</DD> +</DL> + +<DL> + <DT>Property list</DT> + <DD> + <P>An HDF5 API construct, a means of customizing the behavior of the HDF5 + library when creating, accessing or modifying HDF5 items.</P> + <P>While the default property settings are sufficient in many cases, certain + HDF5 features, such as compression, can be reasonably controlled only by the + user who has to provide the desired settings via property lists.</P> +</DD> +</DL> + +\section GLS_R R + +<DL> + <DT>Rank</DT> + <DD>The number of dimensions of a non-null dataspace.</DD> +</DL> + +<DL> + <DT>Reference</DT> + <DD> + <OL> + <LI>An HDF5 object reference</LI> + <LI>An HDF5 dataset region reference</LI> + </OL> + </DD> +</DL> + +<DL> + <DT>Reference datatype</DT> + <DD> + <OL> + <LI>An HDF5 datatype whose elements represent references to HDF5 + objects.</LI> + <LI>An HDF5 datatype whose elements represent references to regions of an + HDF5 dataset.</LI> + </OL> + </DD> +</DL> + +<DL> + <DT>Region reference</DT> + <DD>See dataset region reference.</DD> +</DL> + +<DL> + <DT>Root group</DT> + <DD> + <P>An HDF5 group that is present in all HDF5 files and that acts as the + entry or base point for all other data stored in an HDF5 file.</P> + <P>The root group is "the mother of all objects" in an HDF5 file in the + sense that all objects (and their attributes) can be discovered, + beginning at the root group, by combinations of the following + operations:</P> + <UL> + <LI>Link traversal</LI> + <LI>De-referencing of object references</LI> + </UL> + <P>This discovery is portable and robust with respect to file-internal + storage reorganization.</P> +</DD> +</DL> + +\section GLS_S S + +<DL> + <DT>Scalar dataspace</DT> + <DD>A kind of HDF5 dataspace that has the shape of a singleton, i.e., a set + containing a single element. Array variables with this shape store exactly one + element.</DD> +</DL> + +<DL> + <DT>Selection</DT> + <DD> + <OL> + <LI>A subset of points of an HDF5 dataspace. The subset might be a point + selection or a combination (union, intersection, etc.) of hyperslabs.</LI> + <LI>A subset of dataset elements associated with a dataspace selection as + described under 1.</LI> + </OL> + </DD> +</DL> + +<DL> + <DT>Serialization</DT> + <DD> + <OL> + <LI>The flattening of an N-dimensional array into a 1-dimensional + array.</LI> + <LI>The encoding of a complex data item as a linear byte stream.</LI> + </OL> + </DD> +</DL> + +<DL> + <DT>Soft link</DT> + <DD>A kind of HDF5 link in which the link destination is specified as an HDF5 + path name. The path name may or may not refer to an actual object.</DD> +</DL> + +<DL> + <DT>Storage layout</DT> + <DD>The storage arrangement for dataset elements, links in a group's link + collection, or attributes in an object's attribute collection.</DD> +</DL> + +<DL> + <DT>String datatype</DT> + <DD></DD> +</DL> + +<DL> + <DT>Super block</DT> + <DD>An HDF5 file format primitive; a block of data which contains information + required to access HDF5 files in a portable manner on multiple platforms. The + super block contains information such as version numbers, the size of offsets + and lengths, and the location of the root group.</DD> +</DL> + +<DL> + <DT>SWMR</DT> + <DD>Single Writer Multiple Reader, a file access mode in which a single + process is permitted to write data to an HDF5 file while other processes are + permitted to read data from the same file without the need of inter-process + communication or synchronization.</DD> +</DL> + +<DL> + <DT>Symbolic link</DT> + <DD>An external link or a soft link.</DD> +</DL> + +\section GLS_U U + +<DL> + <DT>User block</DT> + <DD>An HDF5 file format primitive that allows one to set aside a fixed-size + (at least 512 bytes or any power of 2 thereafter) contiguous range of bytes at + the beginning of an HDF5 file for application purposes which will be + skipped/ignored by the HDF5 library.</DD> +</DL> + +<DL> + <DT>UTF-8</DT> + <DD> + <P>A variable-length (1-4 bytes per code point) encoding of the Unicode set + of code points. This is the encoding supported by HDF5 to represent Unicode + strings.</P> + <P>The ASCII encoding is a proper subset of UTF-8.</P> +</DD> +</DL> + +\section GLS_V V + +<DL> + <DT>Variable-length (sequence) datatype</DT> + <DD>A family of HDF5 datatypes whose elements are variable-length sequences of + a given datatype.</DD> +</DL> + +<DL> + <DT>Virtual Dataset (VDS)</DT> + <DD>An HDF5 dataset with virtual storage layout. A dataset whose elements are + partially or entirely stored physically in other datasets.</DD> +</DL> + +<DL> + <DT>Virtual File Driver (VFD)</DT> + <DD></DD> +</DL> + + +<DL> + <DT>Virtual layout</DT> + <DD></DD> +</DL> + + +<DL> + <DT>Virtual Object Layer (VOL)</DT> + <DD></DD> +</DL> + +*/ diff --git a/doxygen/dox/H5AC_cache_config_t.dox b/doxygen/dox/H5AC_cache_config_t.dox index 9b9862b..3faecd5 100644 --- a/doxygen/dox/H5AC_cache_config_t.dox +++ b/doxygen/dox/H5AC_cache_config_t.dox @@ -26,7 +26,7 @@ * * \Emph{*** DEPRECATED ***} Use \Code{H5Fstart/stop} logging functions instead * - * The trace file is a debuging feature that allow the capture of + * The trace file is a debugging feature that allow the capture of * top level metadata cache requests for purposes of debugging and/or * optimization. This field should normally be set to \c FALSE, as * trace file collection imposes considerable overhead. @@ -82,7 +82,7 @@ * H5C_incr__off ) && ( decr_mode == H5C_decr__off )}). There * is no logical reason why this should be so, but it simplifies * implementation and testing, and I can't think of any reason - * why it would be desireable. If you can think of one, I'll + * why it would be desirable. If you can think of one, I'll * revisit the issue. (JM) * \endparblock * @@ -383,7 +383,7 @@ * to disk.\n * When the sync point is reached (or when there is a user generated * flush), process zero flushes sufficient entries to bring it into - * complience with its min clean size (or flushes all dirty entries in + * compliance with its min clean size (or flushes all dirty entries in * the case of a user generated flush), broad casts the list of * entries just cleaned to all the other processes, and then exits * the sync point.\n diff --git a/doxygen/dox/OtherSpecs.dox b/doxygen/dox/OtherSpecs.dox deleted file mode 100644 index e53f26e..0000000 --- a/doxygen/dox/OtherSpecs.dox +++ /dev/null @@ -1,11 +0,0 @@ -/** \page IMG HDF5 Image and Palette Specification Version 1.2 - -\htmlinclude ImageSpec.html - -*/ - -/** \page TBL HDF5 Table Specification Version 1.0 - -\htmlinclude TableSpec.html - -*/ diff --git a/doxygen/dox/Overview.dox b/doxygen/dox/Overview.dox index 754722e..040769c 100644 --- a/doxygen/dox/Overview.dox +++ b/doxygen/dox/Overview.dox @@ -1,14 +1,12 @@ /** \mainpage notitle -This is the documentation set for HDF5. You can -<a href="hdf5-doc.tgz">download</a> it as a tgz archive for offline reading. - -This is the documention set for HDF5 in terms of specifications and software -developed and maintained by <a href="https://www.hdfgroup.org/">The HDF -Group</a>. It is impractical to document the entire HDF5 ecosystem in one place, -and you should also consult the documentation sets of the many outstanding -community projects. +This is the documentation set for HDF5. +It includes specifications and documentation +of software and tools developed and maintained by +<a href="https://www.hdfgroup.org/">The HDF Group</a>. It is impractical to document +the entire HDF5 ecosystem in one place, and you should also consult the documentation +sets of the many outstanding community projects. For a first contact with HDF5, the best place is to have a look at the \link GettingStarted getting started \endlink page that shows you how to write and @@ -19,12 +17,28 @@ technical documentation consists to varying degrees of information related to <em>tasks</em>, <em>concepts</em>, or <em>reference</em> material. As its title suggests, the \link RM Reference Manual \endlink is 100% reference material, while the \link Cookbook \endlink is focused on tasks. The different guide-type -documents cover a mix of tasks, concepts, and reference, to help a certain +documents cover a mix of tasks, concepts, and reference, to help a specific <em>audience</em> succeed. -Finally, do not miss the search engine (top right-hand corner)! If you are -looking for a specific function, it'll take you there directly. If unsure, it'll -give you an idea of what's on offer and a few promising leads. +\par Versions + Version-specific documentation (see the version in the title area) can be found + here: + - HDF5 <code>develop</code> branch (this site) + - <a href="https://docs.hdfgroup.org/hdf5/v1_12/index.html">HDF5 1.12.x</a> + - <a href="https://docs.hdfgroup.org/hdf5/v1_10/index.html">HDF5 1.10.x</a> + - <a href="https://docs.hdfgroup.org/hdf5/v1_8/index.html">HDF5 1.8.x</a> + +\par Search + If you are looking for a specific function, constant, type, etc., use the + search box in the top right-hand corner!\n Otherwise, check out the + \link FTS full-text search\endlink. + +\par Offline reading + You can <a href="hdf5-doc.tgz">download</a> it as a tgz archive for offline reading. + +\par History + A snapshot (~April 2017) of the pre-Doxygen HDF5 documentation can be found + <a href="https://docs.hdfgroup.org/archive/support/HDF5/doc/index.html">here</a>. \par ToDo List There is plenty of <a href="./todo.html">unfinished business</a>. diff --git a/doxygen/dox/Specifications.dox b/doxygen/dox/Specifications.dox index bd7e849..f6fee78 100644 --- a/doxygen/dox/Specifications.dox +++ b/doxygen/dox/Specifications.dox @@ -18,4 +18,40 @@ \li <a href="https://support.hdfgroup.org/HDF5/doc/HL/H5DS_Spec.pdf"> HDF5 Dimension Scale Specification</a> -*/
\ No newline at end of file +*/ + +/** \page FMT3 HDF5 File Format Specification Version 3.0 + +\htmlinclude H5.format.html + +*/ + +/** \page FMT2 HDF5 File Format Specification Version 2.0 + +\htmlinclude H5.format.2.0.html + +*/ + +/** \page FMT11 HDF5 File Format Specification Version 1.1 + +\htmlinclude H5.format.1.1.html + +*/ + +/** \page FMT1 HDF5 File Format Specification Version 1.0 + +\htmlinclude H5.format.1.0.html + +*/ + +/** \page IMG HDF5 Image and Palette Specification Version 1.2 + +\htmlinclude ImageSpec.html + +*/ + +/** \page TBL HDF5 Table Specification Version 1.0 + +\htmlinclude TableSpec.html + +*/ diff --git a/doxygen/dox/TechnicalNotes.dox b/doxygen/dox/TechnicalNotes.dox index 2bda175..9bd2802 100644 --- a/doxygen/dox/TechnicalNotes.dox +++ b/doxygen/dox/TechnicalNotes.dox @@ -1,6 +1,10 @@ /** \page TN Technical Notes \li \link api-compat-macros API Compatibility Macros \endlink +\li \ref APPDBG "Debugging HDF5 Applications" +\li \ref FMTDISC "File Format Walkthrough" +\li \ref FILTER "Filters" +\li \ref IOFLOW "HDF5 Raw I/O Flow Notes" \li \ref TNMDC "Metadata Caching in HDF5" \li \ref MT "Thread Safe library" \li \ref VFL "Virtual File Layer" @@ -13,8 +17,32 @@ */ +/** \page IOFLOW HDF5 Raw I/O Flow Notes + +\htmlinclude IOFlow.html + +*/ + /** \page VFL HDF5 Virtual File Layer \htmlinclude VFL.html */ + +/** \page FMTDISC HDF5 File Format Discussion + +\htmlinclude FileFormat.html + +*/ + +/** \page FILTER HDF5 Filters + +\htmlinclude Filters.html + +*/ + +/** \page APPDBG Debugging HDF5 Applications + +\htmlinclude DebuggingHDF5Applications.html + +*/
\ No newline at end of file diff --git a/doxygen/dox/api-compat-macros.dox b/doxygen/dox/api-compat-macros.dox index 4a1578d..6b506a1 100644 --- a/doxygen/dox/api-compat-macros.dox +++ b/doxygen/dox/api-compat-macros.dox @@ -300,208 +300,6 @@ -DH5Lget_info_vers=1 application.c \endcode - \subsubsection fun-options-112 Function Mapping Options in Releases 1.12.x - <table> - <tr> - <th style="text-align: left;">Macro <br/> (\Code{H5xxx})</th> - <th>Default function used if no macro specified - <ul><li>Function/struct mapping:\Code{H5xxx_vers=N}</li></ul> - </th> - <th>Function used if specifying 1.10 - <ul><li>Function/struct mapping: \Code{H5xxx_vers=1}</li></ul> - </th> - </tr> - <tr> - <td>H5Lget_info()</td> - <td>H5Lget_info2() - <ul> - <li>Function mapping:\Code{H5Lget_info_vers=2}</li> - <li>Struct mapping:\Code{H5L_info_t_vers=2}</li> - </ul> - </td> - <td>H5Lget_info1() - <ul> - <li>Function mapping \Code{H5Lget_info_vers=1}</li> - <li>Struct mapping: \Code{H5L_info_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Lget_info_by_idx()</td> - <td>H5Lget_info_by_idx2() - <ul> - <li>Function mapping: \Code{H5Lget_info_by_idx_vers=2}</li> - <li>Struct mapping: \Code{H5L_info_t_vers=2}</li> - </ul> - </td> - <td>H5Lget_info_by_idx1() - <ul> - <li>Function mapping: \Code{H5Lget_info_by_idx_vers=1}</li> - <li>Struct mapping: \Code{H5L_info_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Literate()</td> - <td>H5Literate2() - <ul> - <li>Function mapping: \Code{H5Literate_vers=2}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=2}</li> - </ul> - </td> - <td>H5Literate1() - <ul> - <li>Function mapping: \Code{H5Literate_vers=1}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Literate_by_name()</td> - <td>H5Literate_by_name2() - <ul> - <li>Function mapping: \Code{H5Literate_by_name_vers=2}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=2}</li> - </ul> - </td> - <td>H5Literate_by_name1() - <ul> - <li>Function mapping: \Code{H5Literate_by_name_vers=1}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Lvisit()</td> - <td>H5Lvisit2() - <ul> - <li>Function mapping: \Code{H5Lvisit_vers=2}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=2}</li> - </ul> - </td> - <td>H5Lvisit1() - <ul> - <li>Function mapping: \Code{H5Lvisit_vers=1}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Lvisit_by_name()</td> - <td>H5Lvisit_by_name2() - <ul> - <li>Function mapping: \Code{H5Lvisit_by_name_vers=2}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=2}</li> - </ul> - </td> - <td>H5Lvisit_by_name1() - <ul> - <li>Function mapping: \Code{H5Lvisit_by_name_vers=1}</li> - <li>Struct mapping: \Code{H5L_iterate_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Oget_info()</td> - <td>H5Oget_info3() - <ul> - <li>Function mapping: \Code{H5Oget_info_vers=3}</li> - <li>Struct mapping: \Code{H5O_info_t_vers=2}</li> - </ul> - </td> - <td>H5Oget_info1() - <ul> - <li>Function mapping: \Code{H5Oget_info_vers=1}</li> - <li>Struct mapping: \Code{H5O_info_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Oget_info_by_idx()</td> - <td>H5Oget_info_by_idx3() - <ul><li>Function mapping: \Code{H5Oget_info_by_idx_vers=3}</li> - <li>Struct mapping: \Code{H5O_info_t_vers=2}</li> - </ul> - </td> - <td>H5Oget_info_by_idx1() - <ul> - <li>Function mapping: \Code{H5Oget_info_by_idx_vers=1}</li> - <li>Struct mapping: \Code{H5O_info_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Oget_info_by_name()</td> - <td>H5Oget_info_by_name3() - <ul> - <li>Function mapping: \Code{H5O_get_info_by_name_vers=3}</li> - <li>Struct mapping: \Code{H5O_info_t_vers=2}</li> - </ul> - </td> - <td>H5Oget_info_by_name1() - <ul> - <li>Function mapping: \Code{H5O_get_info_by_name_vers=1}</li> - <li>Struct mapping: \Code{H5O_info_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Ovisit()</td> - <td>H5Ovisit3() - <ul> - <li>Function mapping: \Code{H5Ovisit_vers=3}</li> - <li>Struct mapping: \Code{H5O_iterate_t_vers=2}</li> - </ul> - </td> - <td>H5Ovisit1() - <ul><li>Function mapping: \Code{H5Ovisit_vers=1}</li> - <li>Struct mapping: \Code{H5O_iterate_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Ovisit_by_name()</td> - <td>H5Ovisit_by_name3() - <ul> - <li>Function mapping: \Code{H5Ovisit_by_name_vers=3}</li> - <li>Struct mapping: \Code{H5O_iterate_t_vers=2}</li> - </ul> - </td> - <td>H5Ovisit_by_name1() - <ul> - <li>Function mapping: \Code{H5Ovisit_by_name_vers=1}</li> - <li>Struct mapping: \Code{H5O_iterate_t_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Pencode()</td> - <td>H5Pencode2() - <ul> - <li>Function mapping: \Code{H5Pencode_vers=2}</li> - </ul> - </td> - <td>H5Pencode1() - <ul> - <li>Function mapping: \Code{H5Pencode_vers=1}</li> - </ul> - </td> - </tr> - <tr> - <td>H5Sencode()</td> - <td>H5Sencode2() - <ul> - <li>Function mapping: \Code{H5Sencode_vers=2}</li> - </ul> - </td> - <td>H5Sencode1() - <ul> - <li>Function mapping: \Code{H5Sencode_vers=1}</li> - </ul> - </td> - </tr> - </table> - \subsubsection fun-options-110 Function Mapping Options in Releases 1.10.x <table> <tr> diff --git a/doxygen/examples/DebuggingHDF5Applications.html b/doxygen/examples/DebuggingHDF5Applications.html new file mode 100644 index 0000000..3390887 --- /dev/null +++ b/doxygen/examples/DebuggingHDF5Applications.html @@ -0,0 +1,392 @@ +<html> + <head> + <title>Debugging HDF5 Applications</title> + + <h2>Introduction</h2> + + <p>The HDF5 library contains a number of debugging features to + make programmers' lives easier including the ability to print + detailed error messages, check invariant conditions, display + timings and other statistics, and trace API function calls and + return values. + + </p><dl> + <dt><b>Error Messages</b> + </dt><dd>Error messages are normally displayed automatically on the + standard error stream and include a stack trace of the library + including file names, line numbers, and function names. The + application has complete control over how error messages are + displayed and can disable the display on a permanent or + temporary basis. Refer to the documentation for the H5E error + handling package. + + <br><br> + </dd><dt><b>Invariant Conditions</b> + </dt><dd>Unless <code>NDEBUG</code> is defined during compiling, the + library will include code to verify that invariant conditions + have the expected values. When a problem is detected the + library will display the file and line number within the + library and the invariant condition that failed. A core dump + may be generated for post mortem debugging. The code to + perform these checks can be included on a per-package bases. + + <br><br> + </dd><dt><b>Timings and Statistics</b> + </dt><dd>The library can be configured to accumulate certain + statistics about things like cache performance, datatype + conversion, data space conversion, and data filters. The code + is included on a per-package basis and enabled at runtime by + an environment variable. + + <br><br> + </dd><dt><b>API Tracing</b> + </dt><dd>All API calls made by an application can be displayed and + include formal argument names and actual values and the + function return value. This code is also conditionally + included at compile time and enabled at runtime. + </dd></dl> + + <p>The statistics and tracing can be displayed on any output + stream (including streams opened by the shell) with output from + different packages even going to different streams. + + </p><h2>Error Messages</h2> + + <p>By default any API function that fails will print an error + stack to the standard error stream. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <p><code></code></p><pre><code> +HDF5-DIAG: Error detected in thread 0. Back trace follows. + #000: H5F.c line 1245 in H5Fopen(): unable to open file + major(04): File interface + minor(10): Unable to open file + #001: H5F.c line 846 in H5F_open(): file does not exist + major(04): File interface + minor(10): Unable to open file + </code></pre> + </td> + </tr> + </tbody></table> + </center> + + <p>The error handling package (H5E) is described + <a href="./group___h5_e.html">elsewhere</a>. + + </p><h2>Invariant Conditions</h2> + + <p>To include checks for invariant conditions the library should + be configured with <code>--disable-production</code>, the + default for versions before 1.2. The library designers have made + every attempt to handle error conditions gracefully but an + invariant condition assertion may fail in certain cases. The + output from a failure usually looks something like this: + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <p><code></code></p><pre><code> +Assertion failed: H5.c:123: i<NELMTS(H5_debug_g) +IOT Trap, core dumped. + </code></pre> + </td> + </tr> + </tbody></table> + </center> + + <h2>Timings and Statistics</h2> + + <p>Code to accumulate statistics is included at compile time by + using the <code>--enable-debug</code> configure switch. The + switch can be followed by an equal sign and a comma-separated + list of package names or else a default list is used. + + </p><p> + </p><center> + <table border="" align="center" width="80%"> + <tbody><tr> + <th>Name</th> + <th>Default</th> + <th>Description</th> + </tr> + <tr> + <td align="center">a</td> + <td align="center">No</td> + <td>Attributes</td> + </tr> + <tr> + <td align="center">ac</td> + <td align="center">Yes</td> + <td>Meta data cache</td> + </tr> + <tr> + <td align="center">b</td> + <td align="center">Yes</td> + <td>B-Trees</td> + </tr> + <tr> + <td align="center">d</td> + <td align="center">Yes</td> + <td>Datasets</td> + </tr> + <tr> + <td align="center">e</td> + <td align="center">Yes</td> + <td>Error handling</td> + </tr> + <tr> + <td align="center">f</td> + <td align="center">Yes</td> + <td>Files</td> + </tr> + <tr> + <td align="center">g</td> + <td align="center">Yes</td> + <td>Groups</td> + </tr> + <tr> + <td align="center">hg</td> + <td align="center">Yes</td> + <td>Global heap</td> + </tr> + <tr> + <td align="center">hl</td> + <td align="center">No</td> + <td>Local heaps</td> + </tr> + <tr> + <td align="center">i</td> + <td align="center">Yes</td> + <td>Interface abstraction</td> + </tr> + <tr> + <td align="center">mf</td> + <td align="center">No</td> + <td>File memory management</td> + </tr> + <tr> + <td align="center">mm</td> + <td align="center">Yes</td> + <td>Library memory management</td> + </tr> + <tr> + <td align="center">o</td> + <td align="center">No</td> + <td>Object headers and messages</td> + </tr> + <tr> + <td align="center">p</td> + <td align="center">Yes</td> + <td>Property lists</td> + </tr> + <tr> + <td align="center">s</td> + <td align="center">Yes</td> + <td>Data spaces</td> + </tr> + <tr> + <td align="center">t</td> + <td align="center">Yes</td> + <td>Datatypes</td> + </tr> + <tr> + <td align="center">v</td> + <td align="center">Yes</td> + <td>Vectors</td> + </tr> + <tr> + <td align="center">z</td> + <td align="center">Yes</td> + <td>Raw data filters</td> + </tr> + </tbody></table> + </center> + + <p>In addition to including the code at compile time the + application must enable each package at runtime. This is done + by listing the package names in the <code>HDF5_DEBUG</code> + environment variable. That variable may also contain file + descriptor numbers (the default is `2') which control the output + for all following packages up to the next file number. The + word <code>all</code> refers to all packages. Any word my be + preceded by a minus sign to turn debugging off for the package. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <caption align="top"><b>Sample debug specifications</b></caption> + <tbody><tr valign="top"> + <td><code>all</code></td> + <td>This causes debugging output from all packages to be + sent to the standard error stream.</td> + </tr> + <tr valign="top"> + <td><code>all -t -s</code></td> + <td>Debugging output for all packages except datatypes + and data spaces will appear on the standard error + stream.</td> + </tr> + <tr valign="top"> + <td><code>-all ac 255 t,s</code></td> + <td>This disables all debugging even if the default was to + debug something, then output from the meta data cache is + send to the standard error stream and output from data + types and spaces is sent to file descriptor 255 which + should be redirected by the shell.</td> + </tr> + </tbody></table> + </center> + + <p>The components of the <code>HDF5_DEBUG</code> value may be + separated by any non-lowercase letter. + + </p><h2>API Tracing</h2> + + <p>The HDF5 library can trace API calls by printing the + function name, the argument names and their values, and the + return value. Some people like to see lots of output during + program execution instead of using a good symbolic debugger, and + this feature is intended for their consumption. For example, + the output from <code>h5ls foo</code> after turning on tracing, + includes: + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <code><pre> +H5Tcopy(type=184549388) = 184549419 (type); +H5Tcopy(type=184549392) = 184549424 (type); +H5Tlock(type=184549424) = SUCCEED; +H5Tcopy(type=184549393) = 184549425 (type); +H5Tlock(type=184549425) = SUCCEED; +H5Fopen(filename="foo", flags=0, access=H5P_DEFAULT) = FAIL; +HDF5-DIAG: Error detected in thread 0. Back trace follows. + #000: H5F.c line 1245 in H5Fopen(): unable to open file + major(04): File interface + minor(10): Unable to open file + #001: H5F.c line 846 in H5F_open(): file does not exist + major(04): File interface + minor(10): Unable to open file + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <p>The code that performs the tracing must be included in the + library by specifying the <code>--enable-trace</code> + configuration switch (the default for versions before 1.2). Then + the word <code>trace</code> must appear in the value of the + <code>HDF5_DEBUG</code> variable. The output will appear on the + last file descriptor before the word <code>trace</code> or two + (standard error) by default. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td>To display the trace on the standard error stream: + <code><pre>$ env HDF5_DEBUG=trace a.out + </pre></code> + </td> + </tr> + <tr> + <td>To send the trace to a file: + <code><pre>$ env HDF5_DEBUG="55 trace" a.out 55>trace-output + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <h3>Performance</h3> + + <p>If the library was not configured for tracing then there is no + unnecessary overhead since all tracing code is excluded. + However, if tracing is enabled but not used there is a small + penalty. First, code size is larger because of extra + statically-declared character strings used to store argument + types and names and extra auto variable pointer in each + function. Also, execution is slower because each function sets + and tests a local variable and each API function calls the + <code>H5_trace()</code> function. + + </p><p>If tracing is enabled and turned on then the penalties from the + previous paragraph apply plus the time required to format each + line of tracing information. There is also an extra call to + H5_trace() for each API function to print the return value. + + </p><h3>Safety</h3> + + <p>The tracing mechanism is invoked for each API function before + arguments are checked for validity. If bad arguments are passed + to an API function it could result in a segmentation fault. + However, the tracing output is line-buffered so all previous + output will appear. + + </p><h3>Completeness</h3> + + <p>There are two API functions that don't participate in + tracing. They are <code>H5Eprint()</code> and + <code>H5Eprint_cb()</code> because their participation would + mess up output during automatic error reporting. + + </p><p>On the other hand, a number of API functions are called during + library initialization and they print tracing information. + + </p><h3>Implementation</h3> + + <p>For those interested in the implementation here is a + description. Each API function should have a call to one of the + <code>H5TRACE()</code> macros immediately after the + <code>FUNC_ENTER()</code> macro. The first argument is the + return type encoded as a string. The second argument is the + types of all the function arguments encoded as a string. The + remaining arguments are the function arguments. This macro was + designed to be as terse and unobtrousive as possible. + + </p><p>In order to keep the <code>H5TRACE()</code> calls synchronized + with the source code we've written a perl script which gets + called automatically just before Makefile dependencies are + calculated for the file. However, this only works when one is + using GNU make. To reinstrument the tracing explicitly, invoke + the <code>trace</code> program from the hdf5 bin directory with + the names of the source files that need to be updated. If any + file needs to be modified then a backup is created by appending + a tilde to the file name. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <caption align="top"><b>Explicit Instrumentation</b></caption> + <tbody><tr> + <td> + <code><pre> +$ ../bin/trace *.c +H5E.c: in function `H5Ewalk_cb': +H5E.c:336: warning: trace info was not inserted + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <p>Note: The warning message is the result of a comment of the + form <code>/*NO TRACE*/</code> somewhere in the function + body. Tracing information will not be updated or inserted if + such a comment exists. + + </p><p>Error messages have the same format as a compiler so that they + can be parsed from program development environments like + Emacs. Any function which generates an error will not be + modified.</p> + +</body></html> diff --git a/doxygen/examples/FileFormat.html b/doxygen/examples/FileFormat.html new file mode 100644 index 0000000..fc35357 --- /dev/null +++ b/doxygen/examples/FileFormat.html @@ -0,0 +1,1275 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<!-- saved from url=(0072)https://gamma.hdfgroup.org/papers/HISS/030515.FileFormat/FileFormat.html --> +<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <title>HDF5 File Format Discussion</title> + + <meta name="author" content="Quincey Koziol"> +</head> + +<body text="#000000" bgcolor="#FFFFFF"> + +<style type="text/css"> +OL.loweralpha { list-style-type: lower-alpha } +OL.upperroman { list-style-type: upper-roman } +</style> + +<style type="text/css"> +TD CAPTION EM { color: red } +TD EM { color: blue } +TABLE CAPTION STRONG { font-size: larger } +</style> + +<center><h1>HDF5 File Format Discussion</h1></center> +<center><h3>Quincey Koziol<br> + koziol@ncsa.uiuc.edu<br> + May 15, 2003 +</h3></center> + +<ol class="upperroman"> + +<li><h3><u>Document's Audience:</u></h3> + +<ul> + <li>Current H5 library designers and knowledgable external developers.</li> +</ul> + +</li><li><h3><u>Background Reading:</u></h3> + +<dl> + <dt><a href="https://docs.hdfgroup.org/hdf5/develop/_s_p_e_c.html">HDF5 File Format Specification</a> + </dt><dd>This describes the current HDF5 file format. +</dd></dl> + +</li><li><h3><u>Introduction:</u></h3> + +<dl> + <dt><strong>What is this document about?</strong></dt> + <dd>This document attempts to explain the HDF5 file format + specification with a few examples and describes some potential + improvements to the format specification. + </dd> <br> +</dl> + +</li><li><h3><u>File Format Examples:</u></h3> + +<p>This section has several small programs and describes the format of a file +created with each of them. +</p> + +<p>Example program one - <em>Create an empty file</em>: +</p><pre> <code> +#include "hdf5.h" +#include <assert.h> + +int main() +{ + hid_t fid; /* File ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +</assert.h></code> </pre> + + <center> + <table border="" align="center" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Super Block</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>\211</td> + <td>'H'</td> + <td>'D'</td> + <td>'F'</td> + </tr> + + <tr align="center"> + <td>\r</td> + <td>\n</td> + <td>\032</td> + <td>\n</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>0</td> + <td>0</td> + <td>0</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>8</td> + <td>8</td> + <td>0</td> + </tr> + + <tr align="center"> + <td colspan="2">4</td> + <td colspan="2">16</td> + </tr> + + <tr align="center"> + <td colspan="4">0x00000003</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">?</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>928<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">H5G_CACHED_STAB (1)</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example1.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached information: + B-tree address: 384 + Heap address: 96 +</code> </pre> + + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1" width="25%">1</td> + <td colspan="1" width="25%">0</td> + <td colspan="2" width="50%">2</td> + </tr> + <tr align="center"> + <td colspan="4">1</td> + </tr> + <tr align="center"> + <td colspan="4">32</td> + </tr> + <tr align="center"> + <td colspan="2">0x0011</td> + <td colspan="2">16</td> + </tr> + <tr align="center"> + <td>0x01</td> + <td colspan="3">0</td> + </tr> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + <tr align="center"> + <td colspan="2">0</td> + <td colspan="2">0</td> + </tr> + <tr align="center"> + <td>0x00</td> + <td colspan="3">0</td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID (sequence number): 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Local Heap</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>'H'</td> + <td>'E'</td> + <td>'A'</td> + <td>'P'</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">256</td> + </tr> + + <tr align="center"> + <td colspan="4">8</td> + </tr> + + <tr align="center"> + <td colspan="4">128</td> + </tr> + </tbody></table> + </center> +<br> + +<pre> <code> +%h5debug example1.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 8, 248 +Percent of heap used: 3.12% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 __ __ __ __ __ __ __ __ ........ + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'T'</td> + <td>'R'</td> + <td>'E'</td> + <td>'E'</td> + + </tr><tr align="center"> + <td>0</td> + <td>0</td> + <td colspan="2">0</td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 0 (32) + +</code> </pre> + +<p></p> + +<p>Example program two - <em>Create a file with a single dataset in it</em>: +</p><pre> <code> +#include "hdf5.h" +#include <assert.h> + +int main() +{ + hid_t fid; /* File ID */ + hid_t sid; /* Dataspace ID */ + hid_t did; /* Dataset ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Create a scalar dataspace for the dataset */ + sid=H5Screate(H5S_SCALAR); + assert(sid>=0); + + /* Create a trivial dataset */ + did=H5Dcreate(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT); + assert(did>=0); + + /* Close the dataset */ + ret=H5Dclose(did); + assert(ret>=0); + + /* Close the dataspace */ + ret=H5Sclose(sid); + assert(ret>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +</assert.h></code> </pre> + + <center> + <table border="" align="center" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Super Block</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>\211</td> + <td>'H'</td> + <td>'D'</td> + <td>'F'</td> + </tr> + + <tr align="center"> + <td>\r</td> + <td>\n</td> + <td>\032</td> + <td>\n</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>0</td> + <td>0</td> + <td>0</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>8</td> + <td>8</td> + <td>0</td> + </tr> + + <tr align="center"> + <td colspan="2">4</td> + <td colspan="2">16</td> + </tr> + + <tr align="center"> + <td colspan="4">0x00000003</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">?</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>928<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">H5G_CACHED_STAB (1)</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example2.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached entry information: + B-tree address: 384 + Heap address: 96 +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1" width="25%">1</td> + <td colspan="1" width="25%">0</td> + <td colspan="2" width="50%">2</td> + </tr> + <tr align="center"> + <td colspan="4">1</td> + </tr> + <tr align="center"> + <td colspan="4">32</td> + </tr> + <tr align="center"> + <td colspan="2">0x0011</td> + <td colspan="2">16</td> + </tr> + <tr align="center"> + <td>0x01</td> + <td colspan="3">0</td> + </tr> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + <tr align="center"> + <td colspan="2">0</td> + <td colspan="2">0</td> + </tr> + <tr align="center"> + <td>0x00</td> + <td colspan="3">0</td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID: 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID: 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Local Heap</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>'H'</td> + <td>'E'</td> + <td>'A'</td> + <td>'P'</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">256</td> + </tr> + + <tr align="center"> + <td colspan="4">16</td> + </tr> + + <tr align="center"> + <td colspan="4">128</td> + </tr> + </tbody></table> + </center> +<br> + +<pre> <code> +%h5debug example2.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 16, 240 +Percent of heap used: 6.25% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 44 61 74 61 73 65 74 00 ........Dataset. + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'T'</td> + <td>'R'</td> + <td>'E'</td> + <td>'E'</td> + + </tr><tr align="center"> + <td>0</td> + <td>0</td> + <td colspan="2">1</td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>1248<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>8<br><br></td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 1 (32) +Child 0... + Address: 1248 + Left Key: + Heap offset: 0 + Name : + Right Key: + Heap offset: 8 + Name : Dataset +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree Symbol Table Node</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'S'</td> + <td>'N'</td> + <td>'O'</td> + <td>'D'</td> + + </tr><tr align="center"> + <td>1</td> + <td>0</td> + <td colspan="2">1</td> + + </tr><tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">8</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>976<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br><br>0<br><br><br></td> + </tr> + </tbody></table> + </td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 1248 96 + +New address: 1248 +Reading signature at address 1248 (rel) +Symbol Table Node... +Dirty: No +Size of Node (in bytes): 328 +Number of Symbols: 1 of 8 +Symbol 0: + Name: `Dataset' + Name offset into private heap: 8 + Object header address: 976 + Dirty: No + Cache info type: Nothing Cached +</code> </pre> + + <center> + <table border="" cellpadding="4" width="90%"> + <caption align="top"> + <strong>'/Dataset' Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1"><em>Version:</em> 1</td> + <td colspan="1"><em>Reserved:</em> 0</td> + <td colspan="2"><em>Number of Header Messages:</em> 6</td> + </tr> + <tr align="center"> + <td colspan="4"><em>Object Reference Count:</em> 1</td> + </tr> + <tr align="center"> + <td colspan="4"><em>Total Object Header Size:</em> 256</td> + </tr> +<!-- Fill Value Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top" color="#80FFFF"> + <em>Fill Value Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0005</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x01</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Space Allocation Time --> + <td colspan="1"><em>Space Allocation Time:</em> 2 (Late)</td> + <!-- Fill Value Writing Time --> + <td colspan="1"><em>Fill Value Writing Time:</em> 0 (At allocation)</td> + <!-- Fill Value Defined --> + <td colspan="1"><em>Fill Value Defined:</em> 0 (Undefined)</td> + </tr> + <tr align="center"> + <!-- Fill Value Datatype Size --> + <td colspan="4"><em>Fill Value Datatype Size:</em> 0 (Use dataset's datatype for fill-value datatype)</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Datatype Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Datatype Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0003</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 16</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x01</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Type Class and Version --> + <td colspan="1"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td width="50%"><em>Version:</em> 0x1</td> + <td><em>Class:</em> 0x0 (Fixed-Point)</td> + </tr> + </tbody></table> + </td> + <!-- Class Bit Field --> + <td colspan="3"><em>Fixed-Point Bit-Field:</em> 0x08 (Little-endian, No padding, Signed)</td> + </tr> + <tr align="center"> + <!-- Type Size (in bytes) --> + <td colspan="4"><em>Size:</em> 4</td> + </tr> + <tr align="center"> + <!-- Bit Offset --> + <td colspan="2"><em>Bit Offset:</em> 0</td> + <!-- Bit Precision --> + <td colspan="2"><em>Bit Precision:</em> 32</td> + </tr> + <tr align="center"> + <!-- Message alignment filler --> + <td colspan="4"><em>Message Alignment Filler:</em> -</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Dataspace Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Dataspace Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0001</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Rank --> + <td colspan="1"><em>Rank:</em> 0 (Scalar)</td> + <!-- Flags --> + <td colspan="1"><em>Flags:</em> 0x00 (No maximum dimensions, no permutation information)</td> + <!-- Reserved --> + <td colspan="1"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Reserved --> + <td colspan="4"><em>Reserved:</em> 0</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Layout Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Layout Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0008</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 24</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Rank --> + <td colspan="1"><em>Rank:</em> 1 (Dataspace rank+1)</td> + <!-- Class --> + <td colspan="1"><em>Class:</em> 1 (Contiguous)</td> + <!-- Reserved --> + <td colspan="1"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Reserved --> + <td colspan="4"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Address --> + <td colspan="4"><br><em>Address:</em> 0xffffffffffffffff (Undefined)<br><br></td> + </tr> + <tr align="center"> + <!-- Layout Dimensions --> + <td colspan="4"><em>Dimension 0 Size:</em> 4 (Datatype size)</td> + </tr> + <tr align="center"> + <!-- Message alignment filler --> + <td colspan="4"><em>Message Alignment Filler:</em> -</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Modification Date & Time Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Modification Date & Time Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0012</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Modification time --> + <td colspan="4"><em>Seconds Since Epoch:</em> 1052401700 (2003-05-08 08:48:20 CDT)</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Null Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Null Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <td colspan="2"><em>Message Type:</em> 0x0000</td> + <td colspan="2"><em>Message Data Size:</em> 144</td> + </tr> + <tr align="center"> + <td colspan="1"><em>Flags:</em> 0x00</td> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 976 + +New address: 976 +Reading signature at address 976 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 6 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 992 + Size in bytes: 256 +Message 0... + Message ID (sequence number): 0x0005 `fill_new' (0) + Shared: No + Constant: Yes + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Version: 1 + Space Allocation Time: Late + Fill Time: On Allocation + Fill Value Defined: Undefined + Size: 0 + Data type: <dataset type=""> +Message 1... + Message ID (sequence number): 0x0003 data_type(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + Type class: integer + Size: 4 bytes + Byte order: little endian + Precision: 32 bits + Offset: 0 bits + Low pad type: zero + High pad type: zero + Sign scheme: 2's comp +Message 2... + Message ID (sequence number): 0x0001 simple_dspace(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Rank: 0 +Message 3... + Message ID (sequence number): 0x0008 layout(0) + Shared message: No + Constant: No + Raw size in obj header: 24 bytes + Chunk number: 0 + Message Information: + Data address: UNDEF + Number of dimensions: 1 + Size: {4} +Message 4... + Message ID (sequence number): 0x0012 mtime_new(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Time: 2003-03-05 14:52:00 CST +Message 5... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 144 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></dataset></code> </pre> + +<p></p> + +</li></ol> + + + +</body></html> diff --git a/doxygen/examples/Filters.html b/doxygen/examples/Filters.html new file mode 100644 index 0000000..7054a3b --- /dev/null +++ b/doxygen/examples/Filters.html @@ -0,0 +1,450 @@ +<html> + <head> + <title>Filters</title> + <h1>Filters in HDF5</h1> + + <b>Note: Transient pipelines described in this document have not + been implemented.</b> + + <h2>Introduction</h2> + + <p>HDF5 allows chunked data to pass through user-defined filters + on the way to or from disk. The filters operate on chunks of an + <code>H5D_CHUNKED</code> dataset can be arranged in a pipeline + so output of one filter becomes the input of the next filter. + + </p><p>Each filter has a two-byte identification number (type + <code>H5Z_filter_t</code>) allocated by The HDF Group and can also be + passed application-defined integer resources to control its + behavior. Each filter also has an optional ASCII comment + string. + + </p> + <table> + <tbody><tr> + <th>Values for <code>H5Z_filter_t</code></th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td><code>0-255</code></td> + <td>These values are reserved for filters predefined and + registered by the HDF5 library and of use to the general + public. They are described in a separate section + below.</td> + </tr> + + <tr valign="top"> + <td><code>256-511</code></td> + <td>Filter numbers in this range are used for testing only + and can be used temporarily by any organization. No + attempt is made to resolve numbering conflicts since all + definitions are by nature temporary.</td> + </tr> + + <tr valign="top"> + <td><code>512-65535</code></td> + <td>Reserved for future assignment. Please contact the + <a href="mailto:help@hdfgroup.org">HDF5 development team</a> + to reserve a value or range of values for + use by your filters.</td> + </tr></tbody></table> + + <h2>Defining and Querying the Filter Pipeline</h2> + + <p>Two types of filters can be applied to raw data I/O: permanent + filters and transient filters. The permanent filter pipeline is + defined when the dataset is created while the transient pipeline + is defined for each I/O operation. During an + <code>H5Dwrite()</code> the transient filters are applied first + in the order defined and then the permanent filters are applied + in the order defined. For an <code>H5Dread()</code> the + opposite order is used: permanent filters in reverse order, then + transient filters in reverse order. An <code>H5Dread()</code> + must result in the same amount of data for a chunk as the + original <code>H5Dwrite()</code>. + + </p><p>The permanent filter pipeline is defined by calling + <code>H5Pset_filter()</code> for a dataset creation property + list while the transient filter pipeline is defined by calling + that function for a dataset transfer property list. + + </p><dl> + <dt><code>herr_t H5Pset_filter (hid_t <em>plist</em>, + H5Z_filter_t <em>filter</em>, unsigned int <em>flags</em>, + size_t <em>cd_nelmts</em>, const unsigned int + <em>cd_values</em>[])</code> + </dt><dd>This function adds the specified <em>filter</em> and + corresponding properties to the end of the transient or + permanent output filter pipeline (depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list). The <em>flags</em> argument specifies certain + general properties of the filter and is documented below. The + <em>cd_values</em> is an array of <em>cd_nelmts</em> integers + which are auxiliary data for the filter. The integer values + will be stored in the dataset object header as part of the + filter information. + </dd><dt><code>int H5Pget_nfilters (hid_t <em>plist</em>)</code> + </dt><dd>This function returns the number of filters defined in the + permanent or transient filter pipeline depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list. In each pipeline the filters are numbered from + 0 through <em>N</em>-1 where <em>N</em> is the value returned + by this function. During output to the file the filters of a + pipeline are applied in increasing order (the inverse is true + for input). Zero is returned if there are no filters in the + pipeline and a negative value is returned for errors. + </dd><dt><code>H5Z_filter_t H5Pget_filter (hid_t <em>plist</em>, + int <em>filter_number</em>, unsigned int *<em>flags</em>, + size_t *<em>cd_nelmts</em>, unsigned int + *<em>cd_values</em>, size_t namelen, char name[])</code> + </dt><dd>This is the query counterpart of + <code>H5Pset_filter()</code> and returns information about a + particular filter number in a permanent or transient pipeline + depending on whether <em>plist</em> is a dataset creation or + dataset transfer property list. On input, <em>cd_nelmts</em> + indicates the number of entries in the <em>cd_values</em> + array allocated by the caller while on exit it contains the + number of values defined by the filter. The + <em>filter_number</em> should be a value between zero and + <em>N</em>-1 as described for <code>H5Pget_nfilters()</code> + and the function will return failure (a negative value) if the + filter number is out of range. If <em>name</em> is a pointer + to an array of at least <em>namelen</em> bytes then the filter + name will be copied into that array. The name will be null + terminated if the <em>namelen</em> is large enough. The + filter name returned will be the name appearing in the file or + else the name registered for the filter or else an empty string. + </dd></dl> + + <p>The flags argument to the functions above is a bit vector of + the following fields: + + </p> + <table> + <tbody><tr> + <th>Values for <em>flags</em></th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td><code>H5Z_FLAG_OPTIONAL</code></td> + <td>If this bit is set then the filter is optional. If + the filter fails (see below) during an + <code>H5Dwrite()</code> operation then the filter is + just excluded from the pipeline for the chunk for which + it failed; the filter will not participate in the + pipeline during an <code>H5Dread()</code> of the chunk. + This is commonly used for compression filters: if the + compression result would be larger than the input then + the compression filter returns failure and the + uncompressed data is stored in the file. If this bit is + clear and a filter fails then the + <code>H5Dwrite()</code> or <code>H5Dread()</code> also + fails.</td> + </tr> + </tbody></table> + + <h2>Defining Filters</h2> + + <p>Each filter is bidirectional, handling both input and output to + the file, and a flag is passed to the filter to indicate the + direction. In either case the filter reads a chunk of data from + a buffer, usually performs some sort of transformation on the + data, places the result in the same or new buffer, and returns + the buffer pointer and size to the caller. If something goes + wrong the filter should return zero to indicate a failure. + + </p><p>During output, a filter that fails or isn't defined and is + marked as optional is silently excluded from the pipeline and + will not be used when reading that chunk of data. A required + filter that fails or isn't defined causes the entire output + operation to fail. During input, any filter that has not been + excluded from the pipeline during output and fails or is not + defined will cause the entire input operation to fail. + + </p><p>Filters are defined in two phases. The first phase is to + define a function to act as the filter and link the function + into the application. The second phase is to register the + function, associating the function with an + <code>H5Z_filter_t</code> identification number and a comment. + + </p><dl> + <dt><code>typedef size_t (*H5Z_func_t)(unsigned int + <em>flags</em>, size_t <em>cd_nelmts</em>, const unsigned int + <em>cd_values</em>[], size_t <em>nbytes</em>, size_t + *<em>buf_size</em>, void **<em>buf</em>)</code> + </dt><dd>The <em>flags</em>, <em>cd_nelmts</em>, and + <em>cd_values</em> are the same as for the + <code>H5Pset_filter()</code> function with the additional flag + <code>H5Z_FLAG_REVERSE</code> which is set when the filter is + called as part of the input pipeline. The input buffer is + pointed to by <em>*buf</em> and has a total size of + <em>*buf_size</em> bytes but only <em>nbytes</em> are valid + data. The filter should perform the transformation in place if + possible and return the number of valid bytes or zero for + failure. If the transformation cannot be done in place then + the filter should allocate a new buffer with + <code>malloc()</code> and assign it to <em>*buf</em>, + assigning the allocated size of that buffer to + <em>*buf_size</em>. The old buffer should be freed + by calling <code>free()</code>. + + <br><br> + </dd><dt><code>herr_t H5Zregister (H5Z_filter_t <em>filter_id</em>, + const char *<em>comment</em>, H5Z_func_t + <em>filter</em>)</code> + </dt><dd>The <em>filter</em> function is associated with a filter + number and a short ASCII comment which will be stored in the + hdf5 file if the filter is used as part of a permanent + pipeline during dataset creation. + </dd></dl> + + <h2>Predefined Filters</h2> + + <p>If <code>zlib</code> version 1.1.2 or later was found + during configuration then the library will define a filter whose + <code>H5Z_filter_t</code> number is + <code>H5Z_FILTER_DEFLATE</code>. Since this compression method + has the potential for generating compressed data which is larger + than the original, the <code>H5Z_FLAG_OPTIONAL</code> flag + should be turned on so such cases can be handled gracefully by + storing the original data instead of the compressed data. The + <em>cd_nvalues</em> should be one with <em>cd_value[0]</em> + being a compression aggression level between zero and nine, + inclusive (zero is the fastest compression while nine results in + the best compression ratio). + + </p><p>A convenience function for adding the + <code>H5Z_FILTER_DEFLATE</code> filter to a pipeline is: + + </p><dl> + <dt><code>herr_t H5Pset_deflate (hid_t <em>plist</em>, unsigned + <em>aggression</em>)</code> + </dt><dd>The deflate compression method is added to the end of the + permanent or transient filter pipeline depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list. The <em>aggression</em> is a number between + zero and nine (inclusive) to indicate the tradeoff between + speed and compression ratio (zero is fastest, nine is best + ratio). + </dd></dl> + + <p>Even if the <code>zlib</code> isn't detected during + configuration the application can define + <code>H5Z_FILTER_DEFLATE</code> as a permanent filter. If the + filter is marked as optional (as with + <code>H5Pset_deflate()</code>) then it will always fail and be + automatically removed from the pipeline. Applications that read + data will fail only if the data is actually compressed; they + won't fail if <code>H5Z_FILTER_DEFLATE</code> was part of the + permanent output pipeline but was automatically excluded because + it didn't exist when the data was written. + + </p><p><code>zlib</code> can be acquired from + <code><a href="http://www.cdrom.com/pub/infozip/zlib/"> + http://www.cdrom.com/pub/infozip/zlib/</a></code>. + + </p><h2>Example</h2> + + <p>This example shows how to define and register a simple filter + that adds a checksum capability to the data stream. + + </p><p>The function that acts as the filter always returns zero + (failure) if the <code>md5()</code> function was not detected at + configuration time (left as an exercise for the reader). + Otherwise the function is broken down to an input and output + half. The output half calculates a checksum, increases the size + of the output buffer if necessary, and appends the checksum to + the end of the buffer. The input half calculates the checksum + on the first part of the buffer and compares it to the checksum + already stored at the end of the buffer. If the two differ then + zero (failure) is returned, otherwise the buffer size is reduced + to exclude the checksum. + + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + size_t + md5_filter(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) + { + #ifdef HAVE_MD5 + unsigned char cksum[16]; + + if (flags & H5Z_REVERSE) { + /* Input */ + assert(nbytes>=16); + md5(nbytes-16, *buf, cksum); + + /* Compare */ + if (memcmp(cksum, (char*)(*buf)+nbytes-16, 16)) { + return 0; /*fail*/ + } + + /* Strip off checksum */ + return nbytes-16; + + } else { + /* Output */ + md5(nbytes, *buf, cksum); + + /* Increase buffer size if necessary */ + if (nbytes+16>*buf_size) { + *buf_size = nbytes + 16; + *buf = realloc(*buf, *buf_size); + } + + /* Append checksum */ + memcpy((char*)(*buf)+nbytes, cksum, 16); + return nbytes+16; + } + #else + return 0; /*fail*/ + #endif + } + </code></pre> + </td> + </tr> + </tbody></table> + + <p>Once the filter function is defined it must be registered so + the HDF5 library knows about it. Since we're testing this + filter we choose one of the <code>H5Z_filter_t</code> numbers + from the reserved range. We'll randomly choose 305. + + </p><p> + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + #define FILTER_MD5 305 + herr_t status = H5Zregister(FILTER_MD5, "md5 checksum", md5_filter); + </code></pre> + </td> + </tr> + </tbody></table> + + <p>Now we can use the filter in a pipeline. We could have added + the filter to the pipeline before defining or registering the + filter as long as the filter was defined and registered by time + we tried to use it (if the filter is marked as optional then we + could have used it without defining it and the library would + have automatically removed it from the pipeline for each chunk + written before the filter was defined and registered). + + </p><p> + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + hsize_t chunk_size[3] = {10,10,10}; + H5Pset_chunk(dcpl, 3, chunk_size); + H5Pset_filter(dcpl, FILTER_MD5, 0, 0, NULL); + hid_t dset = H5Dcreate(file, "dset", H5T_NATIVE_DOUBLE, space, dcpl); + </code></pre> + </td> + </tr> + </tbody></table> + + <h2>6. Filter Diagnostics</h2> + + <p>If the library is compiled with debugging turned on for the H5Z + layer (usually as a result of <code>configure + --enable-debug=z</code>) then filter statistics are printed when + the application exits normally or the library is closed. The + statistics are written to the standard error stream and include + two lines for each filter that was used: one for input and one + for output. The following fields are displayed: + + </p><p> + </p> + <table> + <tbody><tr> + <th>Field Name</th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td>Method</td> + <td>This is the name of the method as defined with + <code>H5Zregister()</code> with the characters + "< or ">" prepended to indicate + input or output.</td> + </tr> + + <tr valign="top"> + <td>Total</td> + <td>The total number of bytes processed by the filter + including errors. This is the maximum of the + <em>nbytes</em> argument or the return value. + </td></tr> + + <tr valign="top"> + <td>Errors</td> + <td>This field shows the number of bytes of the Total + column which can be attributed to errors.</td> + </tr> + + <tr valign="top"> + <td>User, System, Elapsed</td> + <td>These are the amount of user time, system time, and + elapsed time in seconds spent in the filter function. + Elapsed time is sensitive to system load. These times + may be zero on operating systems that don't support the + required operations.</td> + </tr> + + <tr valign="top"> + <td>Bandwidth</td> + <td>This is the filter bandwidth which is the total + number of bytes processed divided by elapsed time. + Since elapsed time is subject to system load the + bandwidth numbers cannot always be trusted. + Furthermore, the bandwidth includes bytes attributed to + errors which may significanly taint the value if the + function is able to detect errors without much + expense.</td> + </tr> + </tbody></table> + + <p> + </p> + <table> + <caption align="bottom"> + <b>Example: Filter Statistics</b> + </caption> + <tbody><tr> + <td> + <p><code></code></p><pre><code>H5Z: filter statistics accumulated ov= + er life of library: + Method Total Errors User System Elapsed Bandwidth + ------ ----- ------ ---- ------ ------- --------- + >deflate 160000 40000 0.62 0.74 1.33 117.5 kBs + <deflate 120000 0 0.11 0.00 0.12 1.000 MBs + </code></pre> + </td> + </tr> + </tbody></table> + + <hr> + + + <p><a name="fn1">Footnote 1:</a> Dataset chunks can be compressed + through the use of filters. Developers should be aware that + reading and rewriting compressed chunked data can result in holes + in an HDF5 file. In time, enough such holes can increase the + file size enough to impair application or library performance + when working with that file. See + <a href="https://support.hdfgroup.org/HDF5/doc1.6/Performance.html#Freespace"> + Freespace Management</a> + in the chapter + <a href="https://support.hdfgroup.org/HDF5/doc1.6/Performance.html"> + Performance Analysis and Issues</a>.</p> +</html> diff --git a/doxygen/examples/H5.format.1.0.html b/doxygen/examples/H5.format.1.0.html index 2d3ffbe..d2b6610 100644 --- a/doxygen/examples/H5.format.1.0.html +++ b/doxygen/examples/H5.format.1.0.html @@ -10,12 +10,12 @@ <table border=0 width=90%> <tr> <td valign=top> - <ol type=I> + <ol type="I"> <li><a href="#Intro">Introduction</a> <li><a href="#BootBlock">Disk Format Level 0 - File Signature and Super Block</a> <li><a href="#Group">Disk Format Level 1 - File Infrastructure</a> <font size=-2> - <ol type=A> + <ol type="A"> <li><a href="#Btrees">Disk Format Level 1A - B-link Trees and B-tree Nodes</a> <li><a href="#SymbolTable">Disk Format Level 1B - Group</a> <li><a href="#SymbolTableEntry">Disk Format Level 1C - Group Entry</a> @@ -26,9 +26,9 @@ </font> <li><a href="#DataObject">Disk Format Level 2 - Data Objects</a> <font size=-2> - <ol type=A> + <ol type="A"> <li><a href="#ObjectHeader">Disk Format Level 2a - Data Object Headers</a> - <ol type=1> + <ol type="1"> <li><a href="#NILMessage">Name: NIL</a> <!-- 0x0000 --> <li><a href="#SimpleDataSpace">Name: Simple Dataspace</a> <!-- 0x0001 --> <!-- @@ -41,13 +41,13 @@ </font> </ol> </td><td> </td><td valign=top> - <ol type=I> + <ol type="I" start="4"> <li><a href="#DataObject">Disk Format Level 2 - Data Objects</a> <font size=-2><i>(Continued)</i> - <ol type=A> + <ol type="A"> <li><a href="#ObjectHeader">Disk Format Level 2a - Data Object Headers</a><i>(Continued)</i> - <ol type=1> + <ol type="1" start="6"> <li><a href="#CompactDataStorageMessage">Name: Data Storage - Compact</a> <!-- 0x0006 --> <li><a href="#ExternalFileListMessage">Name: Data Storage - External Data Files</a> <!-- 0x0007 --> <li><a href="#LayoutMessage">Name: Data Storage - Layout</a> <!-- 0x0008 --> @@ -495,7 +495,7 @@ Elena> "Free-space object" <td>End of File Address</td> <td>This is the relative file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if the free list is not used.</td> </tr> @@ -2414,7 +2414,7 @@ Elena> "Free-space object" <td><b>Normalization.</b> The value can be 0 if there is no normalization, 1 if the most significant bit of the mantissa is always set (except for 0.0), and 2 if the most - signficant bit of the mantissa is not stored but is + significant bit of the mantissa is not stored but is implied to be set. The value 3 is reserved and will not appear in this field.</td> </tr> @@ -2916,7 +2916,7 @@ Elena> "Free-space object" <p>The fill value message stores a single data point value which is returned to the application when an uninitialized data point - is read from the dataset. The fill value is interpretted with + is read from the dataset. The fill value is interpreted with the same datatype as the dataset. If no fill value message is present then a fill value of all zero is assumed. @@ -3327,7 +3327,7 @@ Elena> "Free-space object" <p> <center> - <table border align=center cellpadding=4 witdh="80%"> + <table border align=center cellpadding=4 width="80%"> <caption align=top> <b>Filter Pipeline Message</b> </caption> @@ -3386,7 +3386,7 @@ Elena> "Free-space object" <p> <center> - <table border align=center cellpadding=4 witdh="80%"> + <table border align=center cellpadding=4 width="80%"> <caption align=top> <b>Filter Pipeline Message</b> </caption> diff --git a/doxygen/examples/H5.format.1.1.html b/doxygen/examples/H5.format.1.1.html index ebbbe8e..b91ac90 100644 --- a/doxygen/examples/H5.format.1.1.html +++ b/doxygen/examples/H5.format.1.1.html @@ -36,18 +36,18 @@ TABLE.list TD { border:none; } <table border=0 width=90%> <tr> <td valign=top> - <ol type=I> + <ol type="I"> <li><a href="#Intro">Introduction</a> <li><a href="#FileMetaData">Disk Format Level 0 - File Metadata</a> <font size=-2> - <ol type=A> + <ol type="A"> <li><a href="#SuperBlock">Disk Format Level 0A - File Signature and Super Block</a> <li><a href="#DriverInfo">Disk Format Level 0B - File Driver Info</a> </ol> </font> <li><a href="#FileInfra">Disk Format Level 1 - File Infrastructure</a> <font size=-2> - <ol type=A> + <ol type="A"> <li><a href="#Btrees">Disk Format Level 1A - B-link Trees and B-tree Nodes</a> <li><a href="#SymbolTable">Disk Format Level 1B - Group</a> <li><a href="#SymbolTableEntry">Disk Format Level 1C - Group Entry</a> @@ -58,9 +58,9 @@ TABLE.list TD { border:none; } </font> <li><a href="#DataObject">Disk Format Level 2 - Data Objects</a> <font size=-2> - <ol type=A> + <ol type="A"> <li><a href="#ObjectHeader">Disk Format Level 2a - Data Object Headers</a> - <ol type=1> + <ol type="1"> <li><a href="#NILMessage">Name: NIL</a> <!-- 0x0000 --> <li><a href="#SimpleDataSpace">Name: Simple Dataspace</a> <!-- 0x0001 --> <!-- <li><a href="#DataSpaceMessage">Name: Complex Dataspace</a> --> <!-- 0x0002 --> @@ -73,13 +73,13 @@ TABLE.list TD { border:none; } </font> </ol> </td><td> </td><td valign=top> - <ol type=I start=4> + <ol type="I" start="4"> <li><a href="#DataObject">Disk Format Level 2 - Data Objects</a> <font size=-2><i>(Continued)</i> <ol type=A> <li><a href="#ObjectHeader">Disk Format Level 2a - Data Object Headers</a><i>(Continued)</i> - <ol type=1 start=6> + <ol type="1" start="7"> <!-- <li><a href="#CompactDataStorageMessage">Name: Data Storage - Compact</a> --> <!-- 0x0006 --> <li><a href="#ReservedMessage_0006">Name: Reserved - not assigned yet</a> <!-- 0x0006 --> <li><a href="#ExternalFileListMessage">Name: Data Storage - External Data Files</a> <!-- 0x0007 --> @@ -616,7 +616,7 @@ TABLE.list TD { border:none; } <td> <P>This is the absolute file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if space from the free list is not used. </P> @@ -3184,7 +3184,7 @@ TABLE.list TD { border:none; } <td><b>Normalization.</b> The value can be 0 if there is no normalization, 1 if the most significant bit of the mantissa is always set (except for 0.0), and 2 if the most - signficant bit of the mantissa is not stored but is + significant bit of the mantissa is not stored but is implied to be set. The value 3 is reserved and will not appear in this field.</td> </tr> diff --git a/doxygen/examples/H5.format.2.0.html b/doxygen/examples/H5.format.2.0.html index 3653489..4a5fe37 100644 --- a/doxygen/examples/H5.format.2.0.html +++ b/doxygen/examples/H5.format.2.0.html @@ -821,7 +821,7 @@ II.A. Disk Format: Level 0A - Format Signature and Superblock</a></h3> <td> <p>This is the absolute file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if space from the free list is not used. </p> @@ -4910,7 +4910,7 @@ III.F. Disk Format: Level 1F - Fractal Heap</a></h3> enough to store objects greater than 16 bytes or not. If the heap ID length is 18 bytes or smaller, the ‘normal’ tiny heap ID form is used. If the heap ID length is greater than 18 bytes in - length, the “extented” form is used. See format description below + length, the “extended” form is used. See format description below for both sub-types. </p> </td> @@ -7884,7 +7884,7 @@ IV.A.2. Disk Format: Level 2A2 - Data Object Header Messages</a></h3> <tr> <td align="center"><code>3</code></td> - <td>Message stored is not shared, but is sharable. + <td>Message stored is not shared, but is shareable. </td> </tr> diff --git a/doxygen/examples/H5.format.html b/doxygen/examples/H5.format.html index e16805f..cbcb387 100644 --- a/doxygen/examples/H5.format.html +++ b/doxygen/examples/H5.format.html @@ -936,7 +936,7 @@ <td> <p>This is the absolute file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if space from the free list is not used. </p> @@ -8691,7 +8691,7 @@ three rows are needed. <tr> <td align="center"><code>3</code></td> - <td>Message stored is not shared, but is sharable. + <td>Message stored is not shared, but is shareable. </td> </tr> @@ -20202,7 +20202,7 @@ disk address for the chunk.</p> </tr> <tr> - <td><p>Length fo External File Name</p></td> + <td><p>Length of External File Name</p></td> <td><p>This is the length for the external file name. <p>This field exists if bit 0 of <em>flags</em> is set.</p> </p> diff --git a/doxygen/examples/IOFlow.html b/doxygen/examples/IOFlow.html new file mode 100644 index 0000000..6b2c27e --- /dev/null +++ b/doxygen/examples/IOFlow.html @@ -0,0 +1,137 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<!-- saved from url=(0064)https://gamma.hdfgroup.org/papers/HISS/030821.IOFlow/IOFlow.html --> +<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <title>HDF5 Raw I/O Flow Notes</title> + + <meta name="author" content="Quincey Koziol"> +</head> + +<body text="#000000" bgcolor="#FFFFFF"> + +<style type="text/css"> +OL.loweralpha { list-style-type: lower-alpha } +OL.upperroman { list-style-type: upper-roman } +</style> + +<center><h1>HDF5 Raw I/O Flow Notes</h1></center> +<center><h3>Quincey Koziol<br> + koziol@ncsa.uiuc.edu<br> + August 20, 2003 +</h3></center> + +<ol class="upperroman"> + +<li><h3><u>Document's Audience:</u></h3> + +<ul> + <li>Current H5 library designers and knowledgable external developers.</li> +</ul> + +</li><li><h3><u>Background Reading:</u></h3> + +</li><li><h3><u>Introduction:</u></h3> + +<dl> + <dt><strong>What is this document about?</strong></dt> + <dd>This document attempts to supplement the flow charts describing + the flow of control for raw data I/O in the library. + </dd> <br> +</dl> + +</li><li><h3><u>Figures:</u></h3> +<p>The following figures provide the main information:</p> + <table> + <tr><td><img src="IOFlow.gif" alt="High-Level View of Writing Raw Data" style="height:50%;"></td></tr> + <tr><td><img src="IOFlow2.gif" alt="Perform Serial or Parallel I/O" style="height:50%;"></td></tr> + <tr><td><img src="IOFlow3.gif" alt="Gather/Convert/Scatter" style="height:50%;"></td></tr> + </table> + +</li><li><h3><u>Notes From Accompanying Figures:</u></h3> + +<p>This section provides notes to augment the information in the accompanying + figures. +</p> + +<ol> + <li><b>Validate Parameters</b> - Resolve any H5S_ALL parameters + for dataspace selections to actual dataspaces, allocate + conversion buffers, etc. + </li> + + <li><b>Space Allocated in File?</b> - Space may not have been allocated + in the file to store the dataset data, if "late allocation" was chosen + for the allocation time when the dataset was created. + </li> + + <li><b>Allocate & Fill Space</b> - These operations allocate both contiguous + and chunked dataset's space in the file. The chunked dataset space + allocation iterates through all the chunks in the file and allocates + both the B-tree information and the raw data in the file. Because of + the way filters work, fill-values are written out for chunked datasets + as they are allocated, instead of as a separate step. + In parallel + I/O, the chunked dataset allocation can potentially be time-consuming, + since all the raw data in the dataset is allocated from one process. + </li> + + <li><b>Datatype Conversion Needed?</b> - This currently is the deciding + factor between doing "direct I/O" (in serial or parallel) and needing + to perform gather/convert/scatter operations. I believe that MPI + is capable of performing a limited range of type conversions and if so, + we should add support to detect when they can be used. This will + allow more I/O operations to be performed collectively. + </li> + + <li><b>Collective I/O Requested/Allowed?</b> - A user has to both request + that collective I/O occur and also their I/O operation must meet the + requirements that the library sets for supporting collective parallel + I/O: + <ul> + <li>The dataspace must be scalar or simple (which is a no-op really, + since we don't support "complex" dataspaces in the library + currently). + </li> + <li>The selection must be regular. "all" selections + and hyperslab selections that were + made with only one call to H5Sselect_hyperslab() (i.e. not a + hyperslab selection that has been aggregated over multiple + selection calls) are regular. Supporting point and + irregular hyperslab selections are on the "to do" list. + </li> + <li>The dataset must be stored contiguously on disk (as shown in the + figure also). Supporting chunked dataset storage is also + on the "to do" list. + </li> + </ul> + </li> + + <li><b>Build "chunk map"</b> - This step still has some scalability issues + as it creates a data structure that is proportional to the number of + chunks which will be written to, which could potentially be very large. + Building the "chunk map" information incrementally is on the "to do" + list also. + </li> + + <li><b>Perform Chunked I/O</b> - As the figure shows, there is no support + for collective parallel I/O on chunked datasets currently. As noted + earlier, this is on the "to do" list. + </li> + + <li><b>Perform "Direct" Serial I/O</b> - "Direct" serial I/O writes data + from the application's buffer, without any intervening buffer or memory + copies. For maximum efficiency and performance, the elements in the + selections should be adjoining. + </li> + + <li><b>Perform Collective Parallel I/O</b> - This step also writes data + directly from an application buffer, but additionally uses collective + MPI I/O operations to combine the data from each process in the parallel + application in an efficient manner. + </li> +</ol> + +</li></ol> + + + +</body></html> diff --git a/doxygen/examples/ThreadSafeLibrary.html b/doxygen/examples/ThreadSafeLibrary.html index 8daf386..ebb193e 100644 --- a/doxygen/examples/ThreadSafeLibrary.html +++ b/doxygen/examples/ThreadSafeLibrary.html @@ -165,7 +165,7 @@ The structure is defined in <code>H5private.h</code> as: <blockquote> <pre> - /* cancelability structure */ + /* cancellability structure */ typedef struct H5_cancel_struct { int previous_state; unsigned int cancel_count; diff --git a/doxygen/examples/VFL.html b/doxygen/examples/VFL.html index 9776f96..624d942 100644 --- a/doxygen/examples/VFL.html +++ b/doxygen/examples/VFL.html @@ -306,7 +306,7 @@ H5Dread(dataset, type, mspace, fspace, buffer, dxpl); </PRE> <P> -The transfer propery list can be queried in a manner similar to the file +The transfer property list can be queried in a manner similar to the file access property list: the driver provides a function (or functions) to return various information about the transfer property list: @@ -1210,7 +1210,7 @@ Flush all data for file <VAR>file</VAR> to storage. </P> <P> <STRONG>Example:</STRONG> The sec2 driver doesn't cache any data but it also doesn't -extend the Unix file as agressively as it should. Therefore, when finalizing a +extend the Unix file as aggressively as it should. Therefore, when finalizing a file it should write a zero to the last byte of the allocated region so that when reopening the file later the EOF marker will be at least as large as the EOA marker saved in the superblock (otherwise HDF5 will refuse to open the diff --git a/doxygen/hdf5_navtree_hacks.js b/doxygen/hdf5_navtree_hacks.js index 942970c..dda8984 100644 --- a/doxygen/hdf5_navtree_hacks.js +++ b/doxygen/hdf5_navtree_hacks.js @@ -223,7 +223,7 @@ $(document).ready(function() { (function (){ // wait until the first "selected" element has been created try { - // this line will triger an exception if there is no #selected element, i.e., before the tree structure is complete. + // this line will trigger an exception if there is no #selected element, i.e., before the tree structure is complete. document.getElementById("selected").className = "item selected"; // ok, the default tree has been created, we can keep going... diff --git a/doxygen/hdf5doxy_layout.xml b/doxygen/hdf5doxy_layout.xml index fc20aa1..24642b5 100644 --- a/doxygen/hdf5doxy_layout.xml +++ b/doxygen/hdf5doxy_layout.xml @@ -7,11 +7,12 @@ <tab type="user" url="@ref Cookbook" title="Cookbook" /> <tab type="user" url="https://portal.hdfgroup.org/display/HDF5/HDF5+User+Guides" title="User Guides" /> <tab type="user" url="https://portal.hdfgroup.org/display/HDF5/HDF5+Application+Developer%27s+Guide" title="Application Developer's Guide" /> - <tab type="user" url="https://portal.hdfgroup.org/display/HDF5/HDF5+Glossary" title="Glossary" /> + <tab type="user" url="@ref GLS" title="Glossary" /> <tab type="user" url="@ref RM" title="Reference Manual" /> <tab type="user" url="@ref TN" title="Technical Notes" /> <tab type="user" url="@ref SPEC" title="Specifications" /> <tab type="user" url="@ref RFC" title="RFCs" /> + <tab type="user" url="@ref FTS" title="Full-Text Search" /> <tab type="user" url="@ref About" title="About" /> </navindex> diff --git a/doxygen/img/HDF5.png b/doxygen/img/HDF5.png Binary files differnew file mode 100644 index 0000000..0458fa8 --- /dev/null +++ b/doxygen/img/HDF5.png |
