diff options
Diffstat (limited to 'doxygen')
| -rw-r--r-- | doxygen/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | doxygen/Doxyfile.in | 3 | ||||
| -rw-r--r-- | doxygen/aliases | 16 | ||||
| -rw-r--r-- | doxygen/dox/About.dox | 120 | ||||
| -rw-r--r-- | doxygen/dox/FTS.dox | 8 | ||||
| -rw-r--r-- | doxygen/dox/FileFormatSpec.dox | 23 | ||||
| -rw-r--r-- | doxygen/dox/OtherSpecs.dox | 11 | ||||
| -rw-r--r-- | doxygen/dox/RFC.dox | 11 | ||||
| -rw-r--r-- | doxygen/dox/Specifications.dox | 38 | ||||
| -rw-r--r-- | doxygen/dox/TechnicalNotes.dox | 26 | ||||
| -rw-r--r-- | doxygen/examples/DebuggingHDF5Applications.html | 392 | ||||
| -rw-r--r-- | doxygen/examples/FileFormat.html | 1275 | ||||
| -rw-r--r-- | doxygen/examples/Filters.html | 450 | ||||
| -rw-r--r-- | doxygen/examples/IOFlow.html | 137 | ||||
| -rw-r--r-- | doxygen/hdf5doxy_layout.xml | 1 | ||||
| -rw-r--r-- | doxygen/img/IOFlow.gif | bin | 0 -> 57285 bytes | |||
| -rw-r--r-- | doxygen/img/IOFlow2.gif | bin | 0 -> 29805 bytes | |||
| -rw-r--r-- | doxygen/img/IOFlow3.gif | bin | 0 -> 21442 bytes |
18 files changed, 2467 insertions, 45 deletions
diff --git a/doxygen/CMakeLists.txt b/doxygen/CMakeLists.txt index 36ce590..3462d50 100644 --- a/doxygen/CMakeLists.txt +++ b/doxygen/CMakeLists.txt @@ -27,6 +27,7 @@ if (DOXYGEN_FOUND) set (DOXYGEN_EXTERNAL_SEARCH NO) set (DOXYGEN_SEARCHENGINE_URL) set (DOXYGEN_STRIP_FROM_PATH ${HDF5_SOURCE_DIR}) + set (DOXYGEN_STRIP_FROM_INC_PATH ${HDF5_SOURCE_DIR}) set (DOXYGEN_PREDEFINED "H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD") # This configure and individual custom targets work together diff --git a/doxygen/Doxyfile.in b/doxygen/Doxyfile.in index 44d9974..8c871de 100644 --- a/doxygen/Doxyfile.in +++ b/doxygen/Doxyfile.in @@ -179,7 +179,7 @@ STRIP_FROM_PATH = @DOXYGEN_STRIP_FROM_PATH@ # specify the list of include paths that are normally passed to the compiler # using the -I flag. -STRIP_FROM_INC_PATH = +STRIP_FROM_INC_PATH = @DOXYGEN_STRIP_FROM_INC_PATH@ # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't @@ -856,6 +856,7 @@ INPUT_ENCODING = UTF-8 FILE_PATTERNS = H5*public.h \ H5*module.h \ H5FDcore.h \ + H5FDdevelop.h \ H5FDdirect.h \ H5FDfamily.h \ H5FDhdfs.h \ diff --git a/doxygen/aliases b/doxygen/aliases index 06c3445..68efeb7 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -24,6 +24,7 @@ ALIASES += htri_t="Returns zero (false), a positive (true) or a negative (failur ALIASES += api_vers_2{3}="\1() is a macro that is mapped to either \2() or \3().\n\see \ref api-compat-macros" ALIASES += api_vers_3{4}="\1() is a macro that is mapped to either \2() or \3() or \4().\n\see \ref api-compat-macros" +ALIASES += api_vers_4{5}="\1() is a macro that is mapped to either \2() or \3() or \4() or \5().\n\see \ref api-compat-macros" ALIASES += deprecation_note{1}="\deprecated Superseded by \1." @@ -252,10 +253,17 @@ ALIASES += ref_vol_doc="VOL documentation" ################################################################################ ALIASES += ref_rfc20210528="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_multi_thread.pdf\">Multi-Thread HDF5</a>" +ALIASES += ref_rfc20210219="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/selection_io_RFC_210610.pdf\">Selection I/O</a>" +ALIASES += ref_rfc20200213="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_VFD_subfiling_200424.pdf\">VFD Sub-filing</a>" +ALIASES += ref_rfc20200210="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Onion_VFD_RFC_211122.pdf\">Onion VFD</a>" ALIASES += ref_rfc20190923="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/2019-09-23-RFC_VOL_feature_flags.pdf\">Virtual Object Layer (VOL) API Compatibility</a>" +ALIASES += ref_rfc20190715="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/var_len_data_sketch_design_190715.pdf\">Variable-Length Data in HDF5 Sketch Design</a>" ALIASES += ref_rfc20190410="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_VFD_Plugin.docx.pdf\">A Plugin Interface for HDF5 Virtual File Drivers</a>" ALIASES += ref_rfc20181231="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Min_Obj_Headers_181231.pdf\">Dataset Object Header Size</a>" ALIASES += ref_rfc20181220="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/3.2.1_3.2.2_deliverable_181220_v4.pdf\">MS 3.2 – Addressing Scalability: Scalability of open, close, flush CASE STUDY: CGNS Hotspot analysis of CGNS cgp_open</a>" +ALIASES += ref_rfc20180830="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Sparse_Chunks180830.pdf\">Sparse Chunks</a>" +ALIASES += ref_rfc20180829="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/mirror_VFD_RFC_2018-10-05.pdf\">H5FD_MIRROR Virtual File Driver</a>" +ALIASES += ref_rfc20180815="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/splitter_VFD_RFC_180830.pdf\">Splitter_VFD</a>" ALIASES += ref_rfc20180620="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-Chunking%20Functions-2018-06-20-v3.docx.pdf\">Chunk query functionality in HDF5</a>" ALIASES += ref_rfc20180610="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/VFD_SWMR_RFC_200916.pdf\">VFD SWMR</a>" ALIASES += ref_rfc20180321="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-API_Contexts-2018-03-21.docx.pdf\">API Contexts</a>" @@ -298,7 +306,7 @@ ALIASES += ref_rfc20120305="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC%20P ALIASES += ref_rfc20120220="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/h5repack_improve_hyperslab_over_chunked_dataset_v1.pdf\"><tt>h5repack</tt>: Improved Hyperslab selections for Large Chunked Datasets</a>" ALIASES += ref_rfc20120120="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/2012-1-25-Maintainers-guide-for-datatype.docx.pdf\">A Maintainer’s Guide for the Datatype Module in HDF5 Library</a>" ALIASES += ref_rfc20120104="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_actual_io_v4-1_done.docx.pdf\">Actual I/O Mode</a>" -ALIASES += ref_rfc20111119="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-H5Ocompare-review_v6.pdf\">New public functions to handle comparison</a>" +ALIASES += ref_rfc20111119="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-H5Ocompare-review_v6.pdf\">New public functions to handle comparison</a>" ALIASES += ref_rfc20110825="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/2011-08-31-RFC_H5Ocopy_Named_DT_v2.docx.pdf\">Merging Named Datatypes in H5Ocopy()</a>" ALIASES += ref_rfc20110811="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Enhancement_Hyperslab_Selection-1.4.docx.pdf\">Expanding the HDF5 Hyperslab Selection Interface</a>" ALIASES += ref_rfc20110726="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/metadata_aggregation_RFC_v03.docx.pdf\">HDF5 File Space Allocation and Aggregation</a>" @@ -318,8 +326,12 @@ ALIASES += ref_rfc20091218="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RCF_h5d ALIASES += ref_rfc20090907="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Tools_Lib_v2.pdf\">HDF5 Tools Library Functions</a>" ALIASES += ref_rfc20090612="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_h5diff_default_epsilon.pdf\">Default EPSILON values for comparing floating point data</a>" ALIASES += ref_rfc20081218="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_h5diff_NonComparable.pdf\">Reporting of Non-Comparable Datasets by <tt>h5diff</tt></a>" +ALIASES += ref_rfc20081205="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_elink_callback.pdf\">External Link Traversal Callback</a>" +ALIASES += ref_rfc20081030="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_chunk_cache_functions.pdf\">Setting Raw Data Chunk Cache Parameters in HDF5</a>" ALIASES += ref_rfc20080915="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/FileFreeSpacePerformance.pdf\">Performance Report for Free-space Manager</a>" ALIASES += ref_rfc20080904="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/ExternalLinkFileAccessProperty.pdf\">Setting File Access Property List for accessing External Link</a>" +ALIASES += ref_rfc20080728="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Native_Time_Types.pdf\">Native Time Types in HDF5</a>" +ALIASES += ref_rfc20080723="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Special_Values_in_HDF5.pdf\">Special Values in HDF5</a>" ALIASES += ref_rfc20080301="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/DynamicTransformations_RFC.pdf\">Dynamic Transformations to HDF5 Data</a>" ALIASES += ref_rfc20080209="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-Using-SVN-branching-Feb9.pdf\">Using SVN branching to improve software development process at THG</a>" ALIASES += ref_rfc20080206="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-HIS-REL-1.8_Feb6.pdf\">Maintaining the <tt>HISTORY.txt</tt> and <tt>RELEASE.txt</tt> files in HDF5</a>" @@ -327,7 +339,7 @@ ALIASES += ref_rfc20071111="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/AURA-co ALIASES += ref_rfc20071018="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_NaNsHDF5.pdf\"><tt>NaN</tt> detection in HDF5</a>" ALIASES += ref_rfc20070801="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Metadata_Journaling_RFC.pdf\">Metadata Journaling to Improve Crash Survivability</a>" ALIASES += ref_rfc20070413="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/API_Compatibility_RFC.txt.pdf\">API Compatibility Strategies for HDF5</a>" -ALIASES += ref_rfc20070115="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/PrivateHeap.pdf\">A 'Private' Heap for HDF5</a>" +ALIASES += ref_rfc20070115="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/PrivateHeap.pdf\">A "Private" Heap for HDF5</a>" ALIASES += ref_rfc20060623="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/coll_ind_dd6.pdf\">Performance Comparison of Collective I/O and Independent I/O with Derived Datatypes</a>" ALIASES += ref_rfc20060604="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/h5stat_spec_v3_2006-06-04.pdf\"><tt>h5stat</tt> tool</a>" ALIASES += ref_rfc20060505="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Simple%20Performance%20Test%20on%20Fletcher32%20Filter.pdf\">Simple Performance Test on Fletcher32 Filter</a>" diff --git a/doxygen/dox/About.dox b/doxygen/dox/About.dox index 32930a8..0b21fcc 100644 --- a/doxygen/dox/About.dox +++ b/doxygen/dox/About.dox @@ -12,12 +12,118 @@ of <em>documentation done right</em>. \section about_documentation Documentation about Documentation -\li \todo Describe how to add a reference or a new RFC -\li \todo Describe how to add an example -\li \todo Describe how to include plain HTML -\li \todo Describe how to add an API macro -\li \todo Describe the custom commands -\li \todo Describe the S3 bucket layout and update routine -\li \todo Link the RM template +In this section, we describe common documentation maintenance tasks. + +\subsection plain_html Including Plain HTML Pages + +The most common use case for this is the inclusion of older documentation. +New documentation should, whenever possible, be created using Doxygen markdown! + +Use Doxygen's <a href="https://www.doxygen.nl/manual/commands.html#cmdhtmlinclude"><code>htmlinclude</code></a> +special command to include existing plain HTML pages. + +An example from this documentation set can be seen +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/dox/FileFormatSpec.dox">here</a>. + +\subsection new_rm_entry Creating a New Reference Manual Entry + +Please refer to the \ref RMT for guidance on how to create a new reference manual entry. + +\subsubsection new_example Adding and Referencing API Examples + +For each HDF5 module, such as \Code{H5F}, there is an examples source file called +\Code{H5*_examples.c}. For example, the \Code{H5F} API examples are located in +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/examples/H5F_examples.c"> +<code>H5F_examples.c</code></a>. Examples are code blocks marked as Doxygen +<a href="https://www.doxygen.nl/manual/commands.html#cmdsnippet">snippets</a>. +For example, the source code for the H5Fcreate() API sample is located between +the +\verbatim +//! <!-- [create] --> +... +//! <!-- [create] --> +\endverbatim +comments in +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/examples/H5F_examples.c"> +<code>H5F_examples.c</code></a>. + +Add a new API example by adding a new code block enclosed between matching +snippet tags. The name of the tag is usually the function name stripped of +the module prefix. + +The inclusion of such a block of code can then be triggered via Doxygen's +<a href="https://www.doxygen.nl/manual/commands.html#cmdsnippet"><code>snippet</code></a> +special command. For example, the following markup +\verbatim +* \snippet H5F_examples.c create +\endverbatim +yields +\snippet H5F_examples.c create + +\subsubsection api_macro Adding an API Macro + +API macros are handled by the <code>api_vers_2, api_vers_3, api_vers_4</code> +custom commands. The numbers indicate the number of potential API function +mappings. For example, H5Acreate() has two potential mappings, H5Acreate1() and +H5Acreate2(). To trigger the creation of a reference manual entry for H5Acreate() +use the following markup: +\verbatim +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} +\endverbatim +This yields: + +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} + +\subsection custom_commands Creating Custom Commands + +See Doxygen's <a href="https://www.doxygen.nl/manual/custcmd.html">Custom Commands documentation</a> +as a general reference. + +All custom commands for this project are located in the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file in the <a href="https://github.com/HDFGroup/hdf5/tree/develop/doxygen"><tt>doxygen</tt></a> +subdirectory of the <a href="https://github.com/HDFGroup/hdf5">main HDF5 repo</a>. + +The custom commands are grouped in sections. Find a suitable section for your command or +ask for help if unsure! + +\subsection new_rfc Adding a New RFC or Referencing an Existing RFC + +For ease of reference, we define custom commands for each RFC in the <tt>RFCs</tt> section +of the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file. For example the custom command \Code{ref_rfc20141210} can be used to insert a +reference to "RFC: Virtual Object Layer". In other words, the markup +\verbatim +\ref_rfc20141210 +\endverbatim +yields a clickable link: + +\ref_rfc20141210 + +To add a new RFC, add a custom command for the RFC to the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file. The naming convention for the custom command is \Code{ref_rfcYYYYMMDD}, +where \Code{YYYYMMDD} is the ID of the RFC. The URL is composed of the prefix +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/ +\endverbatim +and the name of your RFC file, typically, a PDF file, i.e., the full URL would +be +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/my_great_rfc_name.pdf +\endverbatim + +\subsection hosting How Do Updates and Changes Get Published? + +Currently, the files underlying this documentation website are stored in an +bucket on AWS S3. The top-level bucket is <pre>s3://docs.hdfgroup.org/hdf5/</pre> +There are folders for the <tt>development</tt> branch and all supported release +version. + +Talk to your friendly IT-team if you need write access, or you need someone to +push an updated version for you! + +\todo Make the publication a GitHub action! */
\ No newline at end of file diff --git a/doxygen/dox/FTS.dox b/doxygen/dox/FTS.dox new file mode 100644 index 0000000..9dae7c1 --- /dev/null +++ b/doxygen/dox/FTS.dox @@ -0,0 +1,8 @@ +/** \page FTS Full-Text Search + +\htmlonly +<script async src="https://cse.google.com/cse.js?cx=75c754173f9b90804"></script> +<div class="gcse-search"></div> +\endhtmlonly + +*/
\ No newline at end of file diff --git a/doxygen/dox/FileFormatSpec.dox b/doxygen/dox/FileFormatSpec.dox deleted file mode 100644 index fc10574..0000000 --- a/doxygen/dox/FileFormatSpec.dox +++ /dev/null @@ -1,23 +0,0 @@ -/** \page FMT3 HDF5 File Format Specification Version 3.0 - -\htmlinclude H5.format.html - -*/ - -/** \page FMT2 HDF5 File Format Specification Version 2.0 - -\htmlinclude H5.format.2.0.html - -*/ - -/** \page FMT11 HDF5 File Format Specification Version 1.1 - -\htmlinclude H5.format.1.1.html - -*/ - -/** \page FMT1 HDF5 File Format Specification Version 1.0 - -\htmlinclude H5.format.1.0.html - -*/
\ No newline at end of file diff --git a/doxygen/dox/OtherSpecs.dox b/doxygen/dox/OtherSpecs.dox deleted file mode 100644 index e53f26e..0000000 --- a/doxygen/dox/OtherSpecs.dox +++ /dev/null @@ -1,11 +0,0 @@ -/** \page IMG HDF5 Image and Palette Specification Version 1.2 - -\htmlinclude ImageSpec.html - -*/ - -/** \page TBL HDF5 Table Specification Version 1.0 - -\htmlinclude TableSpec.html - -*/ diff --git a/doxygen/dox/RFC.dox b/doxygen/dox/RFC.dox index c16dcea..c2562b0 100644 --- a/doxygen/dox/RFC.dox +++ b/doxygen/dox/RFC.dox @@ -3,10 +3,17 @@ <table> <tr><th>RFC ID</th><th>Title</th><th>Comments</th></tr> <tr> <td>2021-05-28</td> <td>\ref_rfc20210528</td> <td></td></tr> +<tr> <td>2021-02-19</td> <td>\ref_rfc20210219</td> <td></td></tr> +<tr> <td>2020-02-13</td> <td>\ref_rfc20200213</td> <td></td></tr> +<tr> <td>2020-02-10</td> <td>\ref_rfc20200210</td> <td></td></tr> <tr> <td>2019-09-23</td> <td>\ref_rfc20190923</td> <td></td></tr> +<tr> <td>2019-07-15</td> <td>\ref_rfc20190715</td> <td></td> </tr> <tr> <td>2019-04-10</td> <td>\ref_rfc20190410</td> <td></td> </tr> <tr> <td>2018-12-31</td> <td>\ref_rfc20181231</td> <td></td> </tr> <tr> <td>2018-12-20</td> <td>\ref_rfc20181220</td> <td></td> </tr> +<tr> <td>2018-08-30</td> <td>\ref_rfc20180830</td> <td></td> </tr> +<tr> <td>2018-08-29</td> <td>\ref_rfc20180829</td> <td></td> </tr> +<tr> <td>2018-08-15</td> <td>\ref_rfc20180815</td> <td></td> </tr> <tr> <td>2018-06-20</td> <td>\ref_rfc20180620</td> <td></td> </tr> <tr> <td>2018-06-10</td> <td>\ref_rfc20180610</td> <td></td> </tr> <tr> <td>2018-03-21</td> <td>\ref_rfc20180321</td> <td></td> </tr> @@ -69,8 +76,12 @@ <tr> <td>2009-09-07</td> <td>\ref_rfc20090907</td> <td></td> </tr> <tr> <td>2009-06-12</td> <td>\ref_rfc20090612</td> <td></td> </tr> <tr> <td>2008-12-18</td> <td>\ref_rfc20081218</td> <td></td> </tr> +<tr> <td>2008-12-05</td> <td>\ref_rfc20081205</td> <td></td> </tr> +<tr> <td>2008-10-30</td> <td>\ref_rfc20081030</td> <td></td> </tr> <tr> <td>2008-09-15</td> <td>\ref_rfc20080915</td> <td></td> </tr> <tr> <td>2008-09-04</td> <td>\ref_rfc20080904</td> <td></td> </tr> +<tr> <td>2008-07-28</td> <td>\ref_rfc20080728</td> <td></td> </tr> +<tr> <td>2008-07-23</td> <td>\ref_rfc20080723</td> <td></td> </tr> <tr> <td>2008-03-01</td> <td>\ref_rfc20080301</td> <td></td> </tr> <tr> <td>2008-02-09</td> <td>\ref_rfc20080209</td> <td></td> </tr> <tr> <td>2008-02-06</td> <td>\ref_rfc20080206</td> <td></td> </tr> diff --git a/doxygen/dox/Specifications.dox b/doxygen/dox/Specifications.dox index 4ae48d0..5a36d61 100644 --- a/doxygen/dox/Specifications.dox +++ b/doxygen/dox/Specifications.dox @@ -19,4 +19,40 @@ \li <a href="https://support.hdfgroup.org/HDF5/doc/HL/H5DS_Spec.pdf"> HDF5 Dimension Scale Specification</a> -*/
\ No newline at end of file +*/ + +/** \page FMT3 HDF5 File Format Specification Version 3.0 + +\htmlinclude H5.format.html + +*/ + +/** \page FMT2 HDF5 File Format Specification Version 2.0 + +\htmlinclude H5.format.2.0.html + +*/ + +/** \page FMT11 HDF5 File Format Specification Version 1.1 + +\htmlinclude H5.format.1.1.html + +*/ + +/** \page FMT1 HDF5 File Format Specification Version 1.0 + +\htmlinclude H5.format.1.0.html + +*/ + +/** \page IMG HDF5 Image and Palette Specification Version 1.2 + +\htmlinclude ImageSpec.html + +*/ + +/** \page TBL HDF5 Table Specification Version 1.0 + +\htmlinclude TableSpec.html + +*/ diff --git a/doxygen/dox/TechnicalNotes.dox b/doxygen/dox/TechnicalNotes.dox index 2bda175..0cabdeb 100644 --- a/doxygen/dox/TechnicalNotes.dox +++ b/doxygen/dox/TechnicalNotes.dox @@ -1,6 +1,10 @@ /** \page TN Technical Notes \li \link api-compat-macros API Compatibility Macros \endlink +\li \ref APPDBG "Debugging HDF5 Applications" +\li \ref FMTDISC "File Format Walkthrough" +\li \ref FILTER "Filters" +\li \ref IOFLOW "HDF5 Raw I/O Flow Notes" \li \ref TNMDC "Metadata Caching in HDF5" \li \ref MT "Thread Safe library" \li \ref VFL "Virtual File Layer" @@ -13,8 +17,30 @@ */ +/** \page IOFLOW HDF5 Raw I/O Flow Notes + +\htmlinclude IOFlow.html + +*/ + /** \page VFL HDF5 Virtual File Layer \htmlinclude VFL.html */ + +/** \page FMTDISC HDF5 File Format Discussion + +\htmlinclude FileFormat.html + +/** \page FILTER HDF5 Filters + +\htmlinclude Filters.html + +*/ + +/** \page APPDBG Debugging HDF5 Applications + +\htmlinclude DebuggingHDF5Applications.html + +*/
\ No newline at end of file diff --git a/doxygen/examples/DebuggingHDF5Applications.html b/doxygen/examples/DebuggingHDF5Applications.html new file mode 100644 index 0000000..c6aaf74 --- /dev/null +++ b/doxygen/examples/DebuggingHDF5Applications.html @@ -0,0 +1,392 @@ +<html> + <head> + <title>Debugging HDF5 Applications</title> + + <h2>Introduction</h2> + + <p>The HDF5 library contains a number of debugging features to + make programmers' lives easier including the ability to print + detailed error messages, check invariant conditions, display + timings and other statistics, and trace API function calls and + return values. + + </p><dl> + <dt><b>Error Messages</b> + </dt><dd>Error messages are normally displayed automatically on the + standard error stream and include a stack trace of the library + including file names, line numbers, and function names. The + application has complete control over how error messages are + displayed and can disable the display on a permanent or + temporary basis. Refer to the documentation for the H5E error + handling package. + + <br><br> + </dd><dt><b>Invariant Conditions</b> + </dt><dd>Unless <code>NDEBUG</code> is defined during compiling, the + library will include code to verify that invariant conditions + have the expected values. When a problem is detected the + library will display the file and line number within the + library and the invariant condition that failed. A core dump + may be generated for post mortem debugging. The code to + perform these checks can be included on a per-package bases. + + <br><br> + </dd><dt><b>Timings and Statistics</b> + </dt><dd>The library can be configured to accumulate certain + statistics about things like cache performance, datatype + conversion, data space conversion, and data filters. The code + is included on a per-package basis and enabled at runtime by + an environment variable. + + <br><br> + </dd><dt><b>API Tracing</b> + </dt><dd>All API calls made by an application can be displayed and + include formal argument names and actual values and the + function return value. This code is also conditionally + included at compile time and enabled at runtime. + </dd></dl> + + <p>The statistics and tracing can be displayed on any output + stream (including streams opened by the shell) with output from + different packages even going to different streams. + + </p><h2>Error Messages</h2> + + <p>By default any API function that fails will print an error + stack to the standard error stream. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <p><code></code></p><pre><code> +HDF5-DIAG: Error detected in thread 0. Back trace follows. + #000: H5F.c line 1245 in H5Fopen(): unable to open file + major(04): File interface + minor(10): Unable to open file + #001: H5F.c line 846 in H5F_open(): file does not exist + major(04): File interface + minor(10): Unable to open file + </code></pre> + </td> + </tr> + </tbody></table> + </center> + + <p>The error handling package (H5E) is described + <a href="./group___h5_e.html">elsewhere</a>. + + </p><h2>Invariant Conditions</h2> + + <p>To include checks for invariant conditions the library should + be configured with <code>--disable-production</code>, the + default for versions before 1.2. The library designers have made + every attempt to handle error conditions gracefully but an + invariant condition assertion may fail in certain cases. The + output from a failure usually looks something like this: + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <p><code></code></p><pre><code> +Assertion failed: H5.c:123: i<NELMTS(H5_debug_g) +IOT Trap, core dumped. + </code></pre> + </td> + </tr> + </tbody></table> + </center> + + <h2>Timings and Statistics</h2> + + <p>Code to accumulate statistics is included at compile time by + using the <code>--enable-debug</code> configure switch. The + switch can be followed by an equal sign and a comma-separated + list of package names or else a default list is used. + + </p><p> + </p><center> + <table border="" align="center" width="80%"> + <tbody><tr> + <th>Name</th> + <th>Default</th> + <th>Description</th> + </tr> + <tr> + <td align="center">a</td> + <td align="center">No</td> + <td>Attributes</td> + </tr> + <tr> + <td align="center">ac</td> + <td align="center">Yes</td> + <td>Meta data cache</td> + </tr> + <tr> + <td align="center">b</td> + <td align="center">Yes</td> + <td>B-Trees</td> + </tr> + <tr> + <td align="center">d</td> + <td align="center">Yes</td> + <td>Datasets</td> + </tr> + <tr> + <td align="center">e</td> + <td align="center">Yes</td> + <td>Error handling</td> + </tr> + <tr> + <td align="center">f</td> + <td align="center">Yes</td> + <td>Files</td> + </tr> + <tr> + <td align="center">g</td> + <td align="center">Yes</td> + <td>Groups</td> + </tr> + <tr> + <td align="center">hg</td> + <td align="center">Yes</td> + <td>Global heap</td> + </tr> + <tr> + <td align="center">hl</td> + <td align="center">No</td> + <td>Local heaps</td> + </tr> + <tr> + <td align="center">i</td> + <td align="center">Yes</td> + <td>Interface abstraction</td> + </tr> + <tr> + <td align="center">mf</td> + <td align="center">No</td> + <td>File memory management</td> + </tr> + <tr> + <td align="center">mm</td> + <td align="center">Yes</td> + <td>Library memory managment</td> + </tr> + <tr> + <td align="center">o</td> + <td align="center">No</td> + <td>Object headers and messages</td> + </tr> + <tr> + <td align="center">p</td> + <td align="center">Yes</td> + <td>Property lists</td> + </tr> + <tr> + <td align="center">s</td> + <td align="center">Yes</td> + <td>Data spaces</td> + </tr> + <tr> + <td align="center">t</td> + <td align="center">Yes</td> + <td>Datatypes</td> + </tr> + <tr> + <td align="center">v</td> + <td align="center">Yes</td> + <td>Vectors</td> + </tr> + <tr> + <td align="center">z</td> + <td align="center">Yes</td> + <td>Raw data filters</td> + </tr> + </tbody></table> + </center> + + <p>In addition to including the code at compile time the + application must enable each package at runtime. This is done + by listing the package names in the <code>HDF5_DEBUG</code> + environment variable. That variable may also contain file + descriptor numbers (the default is `2') which control the output + for all following packages up to the next file number. The + word <code>all</code> refers to all packages. Any word my be + preceded by a minus sign to turn debugging off for the package. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <caption align="top"><b>Sample debug specifications</b></caption> + <tbody><tr valign="top"> + <td><code>all</code></td> + <td>This causes debugging output from all packages to be + sent to the standard error stream.</td> + </tr> + <tr valign="top"> + <td><code>all -t -s</code></td> + <td>Debugging output for all packages except datatypes + and data spaces will appear on the standard error + stream.</td> + </tr> + <tr valign="top"> + <td><code>-all ac 255 t,s</code></td> + <td>This disables all debugging even if the default was to + debug something, then output from the meta data cache is + send to the standard error stream and output from data + types and spaces is sent to file descriptor 255 which + should be redirected by the shell.</td> + </tr> + </tbody></table> + </center> + + <p>The components of the <code>HDF5_DEBUG</code> value may be + separated by any non-lowercase letter. + + </p><h2>API Tracing</h2> + + <p>The HDF5 library can trace API calls by printing the + function name, the argument names and their values, and the + return value. Some people like to see lots of output during + program execution instead of using a good symbolic debugger, and + this feature is intended for their consumption. For example, + the output from <code>h5ls foo</code> after turning on tracing, + includes: + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <code><pre> +H5Tcopy(type=184549388) = 184549419 (type); +H5Tcopy(type=184549392) = 184549424 (type); +H5Tlock(type=184549424) = SUCCEED; +H5Tcopy(type=184549393) = 184549425 (type); +H5Tlock(type=184549425) = SUCCEED; +H5Fopen(filename="foo", flags=0, access=H5P_DEFAULT) = FAIL; +HDF5-DIAG: Error detected in thread 0. Back trace follows. + #000: H5F.c line 1245 in H5Fopen(): unable to open file + major(04): File interface + minor(10): Unable to open file + #001: H5F.c line 846 in H5F_open(): file does not exist + major(04): File interface + minor(10): Unable to open file + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <p>The code that performs the tracing must be included in the + library by specifying the <code>--enable-trace</code> + configuration switch (the default for versions before 1.2). Then + the word <code>trace</code> must appear in the value of the + <code>HDF5_DEBUG</code> variable. The output will appear on the + last file descriptor before the word <code>trace</code> or two + (standard error) by default. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td>To display the trace on the standard error stream: + <code><pre>$ env HDF5_DEBUG=trace a.out + </pre></code> + </td> + </tr> + <tr> + <td>To send the trace to a file: + <code><pre>$ env HDF5_DEBUG="55 trace" a.out 55>trace-output + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <h3>Performance</h3> + + <p>If the library was not configured for tracing then there is no + unnecessary overhead since all tracing code is excluded. + However, if tracing is enabled but not used there is a small + penalty. First, code size is larger because of extra + statically-declared character strings used to store argument + types and names and extra auto variable pointer in each + function. Also, execution is slower because each function sets + and tests a local variable and each API function calls the + <code>H5_trace()</code> function. + + </p><p>If tracing is enabled and turned on then the penalties from the + previous paragraph apply plus the time required to format each + line of tracing information. There is also an extra call to + H5_trace() for each API function to print the return value. + + </p><h3>Safety</h3> + + <p>The tracing mechanism is invoked for each API function before + arguments are checked for validity. If bad arguments are passed + to an API function it could result in a segmentation fault. + However, the tracing output is line-buffered so all previous + output will appear. + + </p><h3>Completeness</h3> + + <p>There are two API functions that don't participate in + tracing. They are <code>H5Eprint()</code> and + <code>H5Eprint_cb()</code> because their participation would + mess up output during automatic error reporting. + + </p><p>On the other hand, a number of API functions are called during + library initialization and they print tracing information. + + </p><h3>Implementation</h3> + + <p>For those interested in the implementation here is a + description. Each API function should have a call to one of the + <code>H5TRACE()</code> macros immediately after the + <code>FUNC_ENTER()</code> macro. The first argument is the + return type encoded as a string. The second argument is the + types of all the function arguments encoded as a string. The + remaining arguments are the function arguments. This macro was + designed to be as terse and unobtrousive as possible. + + </p><p>In order to keep the <code>H5TRACE()</code> calls synchronized + with the source code we've written a perl script which gets + called automatically just before Makefile dependencies are + calculated for the file. However, this only works when one is + using GNU make. To reinstrument the tracing explicitly, invoke + the <code>trace</code> program from the hdf5 bin directory with + the names of the source files that need to be updated. If any + file needs to be modified then a backup is created by appending + a tilde to the file name. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <caption align="top"><b>Explicit Instrumentation</b></caption> + <tbody><tr> + <td> + <code><pre> +$ ../bin/trace *.c +H5E.c: in function `H5Ewalk_cb': +H5E.c:336: warning: trace info was not inserted + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <p>Note: The warning message is the result of a comment of the + form <code>/*NO TRACE*/</code> somewhere in the function + body. Tracing information will not be updated or inserted if + such a comment exists. + + </p><p>Error messages have the same format as a compiler so that they + can be parsed from program development environments like + Emacs. Any function which generates an error will not be + modified.</p> + +</body></html> diff --git a/doxygen/examples/FileFormat.html b/doxygen/examples/FileFormat.html new file mode 100644 index 0000000..fc35357 --- /dev/null +++ b/doxygen/examples/FileFormat.html @@ -0,0 +1,1275 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<!-- saved from url=(0072)https://gamma.hdfgroup.org/papers/HISS/030515.FileFormat/FileFormat.html --> +<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <title>HDF5 File Format Discussion</title> + + <meta name="author" content="Quincey Koziol"> +</head> + +<body text="#000000" bgcolor="#FFFFFF"> + +<style type="text/css"> +OL.loweralpha { list-style-type: lower-alpha } +OL.upperroman { list-style-type: upper-roman } +</style> + +<style type="text/css"> +TD CAPTION EM { color: red } +TD EM { color: blue } +TABLE CAPTION STRONG { font-size: larger } +</style> + +<center><h1>HDF5 File Format Discussion</h1></center> +<center><h3>Quincey Koziol<br> + koziol@ncsa.uiuc.edu<br> + May 15, 2003 +</h3></center> + +<ol class="upperroman"> + +<li><h3><u>Document's Audience:</u></h3> + +<ul> + <li>Current H5 library designers and knowledgable external developers.</li> +</ul> + +</li><li><h3><u>Background Reading:</u></h3> + +<dl> + <dt><a href="https://docs.hdfgroup.org/hdf5/develop/_s_p_e_c.html">HDF5 File Format Specification</a> + </dt><dd>This describes the current HDF5 file format. +</dd></dl> + +</li><li><h3><u>Introduction:</u></h3> + +<dl> + <dt><strong>What is this document about?</strong></dt> + <dd>This document attempts to explain the HDF5 file format + specification with a few examples and describes some potential + improvements to the format specification. + </dd> <br> +</dl> + +</li><li><h3><u>File Format Examples:</u></h3> + +<p>This section has several small programs and describes the format of a file +created with each of them. +</p> + +<p>Example program one - <em>Create an empty file</em>: +</p><pre> <code> +#include "hdf5.h" +#include <assert.h> + +int main() +{ + hid_t fid; /* File ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +</assert.h></code> </pre> + + <center> + <table border="" align="center" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Super Block</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>\211</td> + <td>'H'</td> + <td>'D'</td> + <td>'F'</td> + </tr> + + <tr align="center"> + <td>\r</td> + <td>\n</td> + <td>\032</td> + <td>\n</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>0</td> + <td>0</td> + <td>0</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>8</td> + <td>8</td> + <td>0</td> + </tr> + + <tr align="center"> + <td colspan="2">4</td> + <td colspan="2">16</td> + </tr> + + <tr align="center"> + <td colspan="4">0x00000003</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">?</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>928<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">H5G_CACHED_STAB (1)</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example1.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached information: + B-tree address: 384 + Heap address: 96 +</code> </pre> + + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1" width="25%">1</td> + <td colspan="1" width="25%">0</td> + <td colspan="2" width="50%">2</td> + </tr> + <tr align="center"> + <td colspan="4">1</td> + </tr> + <tr align="center"> + <td colspan="4">32</td> + </tr> + <tr align="center"> + <td colspan="2">0x0011</td> + <td colspan="2">16</td> + </tr> + <tr align="center"> + <td>0x01</td> + <td colspan="3">0</td> + </tr> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + <tr align="center"> + <td colspan="2">0</td> + <td colspan="2">0</td> + </tr> + <tr align="center"> + <td>0x00</td> + <td colspan="3">0</td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID (sequence number): 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Local Heap</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>'H'</td> + <td>'E'</td> + <td>'A'</td> + <td>'P'</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">256</td> + </tr> + + <tr align="center"> + <td colspan="4">8</td> + </tr> + + <tr align="center"> + <td colspan="4">128</td> + </tr> + </tbody></table> + </center> +<br> + +<pre> <code> +%h5debug example1.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 8, 248 +Percent of heap used: 3.12% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 __ __ __ __ __ __ __ __ ........ + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'T'</td> + <td>'R'</td> + <td>'E'</td> + <td>'E'</td> + + </tr><tr align="center"> + <td>0</td> + <td>0</td> + <td colspan="2">0</td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 0 (32) + +</code> </pre> + +<p></p> + +<p>Example program two - <em>Create a file with a single dataset in it</em>: +</p><pre> <code> +#include "hdf5.h" +#include <assert.h> + +int main() +{ + hid_t fid; /* File ID */ + hid_t sid; /* Dataspace ID */ + hid_t did; /* Dataset ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Create a scalar dataspace for the dataset */ + sid=H5Screate(H5S_SCALAR); + assert(sid>=0); + + /* Create a trivial dataset */ + did=H5Dcreate(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT); + assert(did>=0); + + /* Close the dataset */ + ret=H5Dclose(did); + assert(ret>=0); + + /* Close the dataspace */ + ret=H5Sclose(sid); + assert(ret>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +</assert.h></code> </pre> + + <center> + <table border="" align="center" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Super Block</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>\211</td> + <td>'H'</td> + <td>'D'</td> + <td>'F'</td> + </tr> + + <tr align="center"> + <td>\r</td> + <td>\n</td> + <td>\032</td> + <td>\n</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>0</td> + <td>0</td> + <td>0</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>8</td> + <td>8</td> + <td>0</td> + </tr> + + <tr align="center"> + <td colspan="2">4</td> + <td colspan="2">16</td> + </tr> + + <tr align="center"> + <td colspan="4">0x00000003</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">?</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>928<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">H5G_CACHED_STAB (1)</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example2.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached entry information: + B-tree address: 384 + Heap address: 96 +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1" width="25%">1</td> + <td colspan="1" width="25%">0</td> + <td colspan="2" width="50%">2</td> + </tr> + <tr align="center"> + <td colspan="4">1</td> + </tr> + <tr align="center"> + <td colspan="4">32</td> + </tr> + <tr align="center"> + <td colspan="2">0x0011</td> + <td colspan="2">16</td> + </tr> + <tr align="center"> + <td>0x01</td> + <td colspan="3">0</td> + </tr> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + <tr align="center"> + <td colspan="2">0</td> + <td colspan="2">0</td> + </tr> + <tr align="center"> + <td>0x00</td> + <td colspan="3">0</td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID: 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID: 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Local Heap</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>'H'</td> + <td>'E'</td> + <td>'A'</td> + <td>'P'</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">256</td> + </tr> + + <tr align="center"> + <td colspan="4">16</td> + </tr> + + <tr align="center"> + <td colspan="4">128</td> + </tr> + </tbody></table> + </center> +<br> + +<pre> <code> +%h5debug example2.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 16, 240 +Percent of heap used: 6.25% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 44 61 74 61 73 65 74 00 ........Dataset. + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'T'</td> + <td>'R'</td> + <td>'E'</td> + <td>'E'</td> + + </tr><tr align="center"> + <td>0</td> + <td>0</td> + <td colspan="2">1</td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>1248<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>8<br><br></td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 1 (32) +Child 0... + Address: 1248 + Left Key: + Heap offset: 0 + Name : + Right Key: + Heap offset: 8 + Name : Dataset +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree Symbol Table Node</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'S'</td> + <td>'N'</td> + <td>'O'</td> + <td>'D'</td> + + </tr><tr align="center"> + <td>1</td> + <td>0</td> + <td colspan="2">1</td> + + </tr><tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">8</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>976<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br><br>0<br><br><br></td> + </tr> + </tbody></table> + </td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 1248 96 + +New address: 1248 +Reading signature at address 1248 (rel) +Symbol Table Node... +Dirty: No +Size of Node (in bytes): 328 +Number of Symbols: 1 of 8 +Symbol 0: + Name: `Dataset' + Name offset into private heap: 8 + Object header address: 976 + Dirty: No + Cache info type: Nothing Cached +</code> </pre> + + <center> + <table border="" cellpadding="4" width="90%"> + <caption align="top"> + <strong>'/Dataset' Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1"><em>Version:</em> 1</td> + <td colspan="1"><em>Reserved:</em> 0</td> + <td colspan="2"><em>Number of Header Messages:</em> 6</td> + </tr> + <tr align="center"> + <td colspan="4"><em>Object Reference Count:</em> 1</td> + </tr> + <tr align="center"> + <td colspan="4"><em>Total Object Header Size:</em> 256</td> + </tr> +<!-- Fill Value Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top" color="#80FFFF"> + <em>Fill Value Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0005</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x01</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Space Allocation Time --> + <td colspan="1"><em>Space Allocation Time:</em> 2 (Late)</td> + <!-- Fill Value Writing Time --> + <td colspan="1"><em>Fill Value Writing Time:</em> 0 (At allocation)</td> + <!-- Fill Value Defined --> + <td colspan="1"><em>Fill Value Defined:</em> 0 (Undefined)</td> + </tr> + <tr align="center"> + <!-- Fill Value Datatype Size --> + <td colspan="4"><em>Fill Value Datatype Size:</em> 0 (Use dataset's datatype for fill-value datatype)</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Datatype Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Datatype Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0003</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 16</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x01</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Type Class and Version --> + <td colspan="1"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td width="50%"><em>Version:</em> 0x1</td> + <td><em>Class:</em> 0x0 (Fixed-Point)</td> + </tr> + </tbody></table> + </td> + <!-- Class Bit Field --> + <td colspan="3"><em>Fixed-Point Bit-Field:</em> 0x08 (Little-endian, No padding, Signed)</td> + </tr> + <tr align="center"> + <!-- Type Size (in bytes) --> + <td colspan="4"><em>Size:</em> 4</td> + </tr> + <tr align="center"> + <!-- Bit Offset --> + <td colspan="2"><em>Bit Offset:</em> 0</td> + <!-- Bit Precision --> + <td colspan="2"><em>Bit Precision:</em> 32</td> + </tr> + <tr align="center"> + <!-- Message alignment filler --> + <td colspan="4"><em>Message Alignment Filler:</em> -</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Dataspace Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Dataspace Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0001</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Rank --> + <td colspan="1"><em>Rank:</em> 0 (Scalar)</td> + <!-- Flags --> + <td colspan="1"><em>Flags:</em> 0x00 (No maximum dimensions, no permutation information)</td> + <!-- Reserved --> + <td colspan="1"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Reserved --> + <td colspan="4"><em>Reserved:</em> 0</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Layout Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Layout Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0008</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 24</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Rank --> + <td colspan="1"><em>Rank:</em> 1 (Dataspace rank+1)</td> + <!-- Class --> + <td colspan="1"><em>Class:</em> 1 (Contiguous)</td> + <!-- Reserved --> + <td colspan="1"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Reserved --> + <td colspan="4"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Address --> + <td colspan="4"><br><em>Address:</em> 0xffffffffffffffff (Undefined)<br><br></td> + </tr> + <tr align="center"> + <!-- Layout Dimensions --> + <td colspan="4"><em>Dimension 0 Size:</em> 4 (Datatype size)</td> + </tr> + <tr align="center"> + <!-- Message alignment filler --> + <td colspan="4"><em>Message Alignment Filler:</em> -</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Modification Date & Time Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Modification Date & Time Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0012</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Modification time --> + <td colspan="4"><em>Seconds Since Epoch:</em> 1052401700 (2003-05-08 08:48:20 CDT)</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Null Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Null Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <td colspan="2"><em>Message Type:</em> 0x0000</td> + <td colspan="2"><em>Message Data Size:</em> 144</td> + </tr> + <tr align="center"> + <td colspan="1"><em>Flags:</em> 0x00</td> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 976 + +New address: 976 +Reading signature at address 976 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 6 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 992 + Size in bytes: 256 +Message 0... + Message ID (sequence number): 0x0005 `fill_new' (0) + Shared: No + Constant: Yes + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Version: 1 + Space Allocation Time: Late + Fill Time: On Allocation + Fill Value Defined: Undefined + Size: 0 + Data type: <dataset type=""> +Message 1... + Message ID (sequence number): 0x0003 data_type(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + Type class: integer + Size: 4 bytes + Byte order: little endian + Precision: 32 bits + Offset: 0 bits + Low pad type: zero + High pad type: zero + Sign scheme: 2's comp +Message 2... + Message ID (sequence number): 0x0001 simple_dspace(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Rank: 0 +Message 3... + Message ID (sequence number): 0x0008 layout(0) + Shared message: No + Constant: No + Raw size in obj header: 24 bytes + Chunk number: 0 + Message Information: + Data address: UNDEF + Number of dimensions: 1 + Size: {4} +Message 4... + Message ID (sequence number): 0x0012 mtime_new(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Time: 2003-03-05 14:52:00 CST +Message 5... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 144 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></dataset></code> </pre> + +<p></p> + +</li></ol> + + + +</body></html> diff --git a/doxygen/examples/Filters.html b/doxygen/examples/Filters.html new file mode 100644 index 0000000..2d5bc5e --- /dev/null +++ b/doxygen/examples/Filters.html @@ -0,0 +1,450 @@ +<html> + <head> + <title>Filters</title> + <h1>Filters in HDF5</h1> + + <b>Note: Transient pipelines described in this document have not + been implemented.</b> + + <h2>Introduction</h2> + + <p>HDF5 allows chunked data to pass through user-defined filters + on the way to or from disk. The filters operate on chunks of an + <code>H5D_CHUNKED</code> dataset can be arranged in a pipeline + so output of one filter becomes the input of the next filter. + + </p><p>Each filter has a two-byte identification number (type + <code>H5Z_filter_t</code>) allocated by The HDF Group and can also be + passed application-defined integer resources to control its + behavior. Each filter also has an optional ASCII comment + string. + + </p> + <table> + <tbody><tr> + <th>Values for <code>H5Z_filter_t</code></th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td><code>0-255</code></td> + <td>These values are reserved for filters predefined and + registered by the HDF5 library and of use to the general + public. They are described in a separate section + below.</td> + </tr> + + <tr valign="top"> + <td><code>256-511</code></td> + <td>Filter numbers in this range are used for testing only + and can be used temporarily by any organization. No + attempt is made to resolve numbering conflicts since all + definitions are by nature temporary.</td> + </tr> + + <tr valign="top"> + <td><code>512-65535</code></td> + <td>Reserved for future assignment. Please contact the + <a href="mailto:help@hdfgroup.org">HDF5 development team</a> + to reserve a value or range of values for + use by your filters.</td> + </tr></tbody></table> + + <h2>Defining and Querying the Filter Pipeline</h2> + + <p>Two types of filters can be applied to raw data I/O: permanent + filters and transient filters. The permanent filter pipeline is + defned when the dataset is created while the transient pipeline + is defined for each I/O operation. During an + <code>H5Dwrite()</code> the transient filters are applied first + in the order defined and then the permanent filters are applied + in the order defined. For an <code>H5Dread()</code> the + opposite order is used: permanent filters in reverse order, then + transient filters in reverse order. An <code>H5Dread()</code> + must result in the same amount of data for a chunk as the + original <code>H5Dwrite()</code>. + + </p><p>The permanent filter pipeline is defined by calling + <code>H5Pset_filter()</code> for a dataset creation property + list while the transient filter pipeline is defined by calling + that function for a dataset transfer property list. + + </p><dl> + <dt><code>herr_t H5Pset_filter (hid_t <em>plist</em>, + H5Z_filter_t <em>filter</em>, unsigned int <em>flags</em>, + size_t <em>cd_nelmts</em>, const unsigned int + <em>cd_values</em>[])</code> + </dt><dd>This function adds the specified <em>filter</em> and + corresponding properties to the end of the transient or + permanent output filter pipeline (depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list). The <em>flags</em> argument specifies certain + general properties of the filter and is documented below. The + <em>cd_values</em> is an array of <em>cd_nelmts</em> integers + which are auxiliary data for the filter. The integer values + will be stored in the dataset object header as part of the + filter information. + </dd><dt><code>int H5Pget_nfilters (hid_t <em>plist</em>)</code> + </dt><dd>This function returns the number of filters defined in the + permanent or transient filter pipeline depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list. In each pipeline the filters are numbered from + 0 through <em>N</em>-1 where <em>N</em> is the value returned + by this function. During output to the file the filters of a + pipeline are applied in increasing order (the inverse is true + for input). Zero is returned if there are no filters in the + pipeline and a negative value is returned for errors. + </dd><dt><code>H5Z_filter_t H5Pget_filter (hid_t <em>plist</em>, + int <em>filter_number</em>, unsigned int *<em>flags</em>, + size_t *<em>cd_nelmts</em>, unsigned int + *<em>cd_values</em>, size_t namelen, char name[])</code> + </dt><dd>This is the query counterpart of + <code>H5Pset_filter()</code> and returns information about a + particular filter number in a permanent or transient pipeline + depending on whether <em>plist</em> is a dataset creation or + dataset transfer property list. On input, <em>cd_nelmts</em> + indicates the number of entries in the <em>cd_values</em> + array allocated by the caller while on exit it contains the + number of values defined by the filter. The + <em>filter_number</em> should be a value between zero and + <em>N</em>-1 as described for <code>H5Pget_nfilters()</code> + and the function will return failure (a negative value) if the + filter number is out of range. If <em>name</em> is a pointer + to an array of at least <em>namelen</em> bytes then the filter + name will be copied into that array. The name will be null + terminated if the <em>namelen</em> is large enough. The + filter name returned will be the name appearing in the file or + else the name registered for the filter or else an empty string. + </dd></dl> + + <p>The flags argument to the functions above is a bit vector of + the following fields: + + </p> + <table> + <tbody><tr> + <th>Values for <em>flags</em></th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td><code>H5Z_FLAG_OPTIONAL</code></td> + <td>If this bit is set then the filter is optional. If + the filter fails (see below) during an + <code>H5Dwrite()</code> operation then the filter is + just excluded from the pipeline for the chunk for which + it failed; the filter will not participate in the + pipeline during an <code>H5Dread()</code> of the chunk. + This is commonly used for compression filters: if the + compression result would be larger than the input then + the compression filter returns failure and the + uncompressed data is stored in the file. If this bit is + clear and a filter fails then the + <code>H5Dwrite()</code> or <code>H5Dread()</code> also + fails.</td> + </tr> + </tbody></table> + + <h2>Defining Filters</h2> + + <p>Each filter is bidirectional, handling both input and output to + the file, and a flag is passed to the filter to indicate the + direction. In either case the filter reads a chunk of data from + a buffer, usually performs some sort of transformation on the + data, places the result in the same or new buffer, and returns + the buffer pointer and size to the caller. If something goes + wrong the filter should return zero to indicate a failure. + + </p><p>During output, a filter that fails or isn't defined and is + marked as optional is silently excluded from the pipeline and + will not be used when reading that chunk of data. A required + filter that fails or isn't defined causes the entire output + operation to fail. During input, any filter that has not been + excluded from the pipeline during output and fails or is not + defined will cause the entire input operation to fail. + + </p><p>Filters are defined in two phases. The first phase is to + define a function to act as the filter and link the function + into the application. The second phase is to register the + function, associating the function with an + <code>H5Z_filter_t</code> identification number and a comment. + + </p><dl> + <dt><code>typedef size_t (*H5Z_func_t)(unsigned int + <em>flags</em>, size_t <em>cd_nelmts</em>, const unsigned int + <em>cd_values</em>[], size_t <em>nbytes</em>, size_t + *<em>buf_size</em>, void **<em>buf</em>)</code> + </dt><dd>The <em>flags</em>, <em>cd_nelmts</em>, and + <em>cd_values</em> are the same as for the + <code>H5Pset_filter()</code> function with the additional flag + <code>H5Z_FLAG_REVERSE</code> which is set when the filter is + called as part of the input pipeline. The input buffer is + pointed to by <em>*buf</em> and has a total size of + <em>*buf_size</em> bytes but only <em>nbytes</em> are valid + data. The filter should perform the transformation in place if + possible and return the number of valid bytes or zero for + failure. If the transformation cannot be done in place then + the filter should allocate a new buffer with + <code>malloc()</code> and assign it to <em>*buf</em>, + assigning the allocated size of that buffer to + <em>*buf_size</em>. The old buffer should be freed + by calling <code>free()</code>. + + <br><br> + </dd><dt><code>herr_t H5Zregister (H5Z_filter_t <em>filter_id</em>, + const char *<em>comment</em>, H5Z_func_t + <em>filter</em>)</code> + </dt><dd>The <em>filter</em> function is associated with a filter + number and a short ASCII comment which will be stored in the + hdf5 file if the filter is used as part of a permanent + pipeline during dataset creation. + </dd></dl> + + <h2>Predefined Filters</h2> + + <p>If <code>zlib</code> version 1.1.2 or later was found + during configuration then the library will define a filter whose + <code>H5Z_filter_t</code> number is + <code>H5Z_FILTER_DEFLATE</code>. Since this compression method + has the potential for generating compressed data which is larger + than the original, the <code>H5Z_FLAG_OPTIONAL</code> flag + should be turned on so such cases can be handled gracefully by + storing the original data instead of the compressed data. The + <em>cd_nvalues</em> should be one with <em>cd_value[0]</em> + being a compression agression level between zero and nine, + inclusive (zero is the fastest compression while nine results in + the best compression ratio). + + </p><p>A convenience function for adding the + <code>H5Z_FILTER_DEFLATE</code> filter to a pipeline is: + + </p><dl> + <dt><code>herr_t H5Pset_deflate (hid_t <em>plist</em>, unsigned + <em>aggression</em>)</code> + </dt><dd>The deflate compression method is added to the end of the + permanent or transient filter pipeline depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list. The <em>aggression</em> is a number between + zero and nine (inclusive) to indicate the tradeoff between + speed and compression ratio (zero is fastest, nine is best + ratio). + </dd></dl> + + <p>Even if the <code>zlib</code> isn't detected during + configuration the application can define + <code>H5Z_FILTER_DEFLATE</code> as a permanent filter. If the + filter is marked as optional (as with + <code>H5Pset_deflate()</code>) then it will always fail and be + automatically removed from the pipeline. Applications that read + data will fail only if the data is actually compressed; they + won't fail if <code>H5Z_FILTER_DEFLATE</code> was part of the + permanent output pipeline but was automatically excluded because + it didn't exist when the data was written. + + </p><p><code>zlib</code> can be acquired from + <code><a href="http://www.cdrom.com/pub/infozip/zlib/"> + http://www.cdrom.com/pub/infozip/zlib/</a></code>. + + </p><h2>Example</h2> + + <p>This example shows how to define and register a simple filter + that adds a checksum capability to the data stream. + + </p><p>The function that acts as the filter always returns zero + (failure) if the <code>md5()</code> function was not detected at + configuration time (left as an excercise for the reader). + Otherwise the function is broken down to an input and output + half. The output half calculates a checksum, increases the size + of the output buffer if necessary, and appends the checksum to + the end of the buffer. The input half calculates the checksum + on the first part of the buffer and compares it to the checksum + already stored at the end of the buffer. If the two differ then + zero (failure) is returned, otherwise the buffer size is reduced + to exclude the checksum. + + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + size_t + md5_filter(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) + { + #ifdef HAVE_MD5 + unsigned char cksum[16]; + + if (flags & H5Z_REVERSE) { + /* Input */ + assert(nbytes>=16); + md5(nbytes-16, *buf, cksum); + + /* Compare */ + if (memcmp(cksum, (char*)(*buf)+nbytes-16, 16)) { + return 0; /*fail*/ + } + + /* Strip off checksum */ + return nbytes-16; + + } else { + /* Output */ + md5(nbytes, *buf, cksum); + + /* Increase buffer size if necessary */ + if (nbytes+16>*buf_size) { + *buf_size = nbytes + 16; + *buf = realloc(*buf, *buf_size); + } + + /* Append checksum */ + memcpy((char*)(*buf)+nbytes, cksum, 16); + return nbytes+16; + } + #else + return 0; /*fail*/ + #endif + } + </code></pre> + </td> + </tr> + </tbody></table> + + <p>Once the filter function is defined it must be registered so + the HDF5 library knows about it. Since we're testing this + filter we choose one of the <code>H5Z_filter_t</code> numbers + from the reserved range. We'll randomly choose 305. + + </p><p> + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + #define FILTER_MD5 305 + herr_t status = H5Zregister(FILTER_MD5, "md5 checksum", md5_filter); + </code></pre> + </td> + </tr> + </tbody></table> + + <p>Now we can use the filter in a pipeline. We could have added + the filter to the pipeline before defining or registering the + filter as long as the filter was defined and registered by time + we tried to use it (if the filter is marked as optional then we + could have used it without defining it and the library would + have automatically removed it from the pipeline for each chunk + written before the filter was defined and registered). + + </p><p> + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + hsize_t chunk_size[3] = {10,10,10}; + H5Pset_chunk(dcpl, 3, chunk_size); + H5Pset_filter(dcpl, FILTER_MD5, 0, 0, NULL); + hid_t dset = H5Dcreate(file, "dset", H5T_NATIVE_DOUBLE, space, dcpl); + </code></pre> + </td> + </tr> + </tbody></table> + + <h2>6. Filter Diagnostics</h2> + + <p>If the library is compiled with debugging turned on for the H5Z + layer (usually as a result of <code>configure + --enable-debug=z</code>) then filter statistics are printed when + the application exits normally or the library is closed. The + statistics are written to the standard error stream and include + two lines for each filter that was used: one for input and one + for output. The following fields are displayed: + + </p><p> + </p> + <table> + <tbody><tr> + <th>Field Name</th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td>Method</td> + <td>This is the name of the method as defined with + <code>H5Zregister()</code> with the charaters + "< or ">" prepended to indicate + input or output.</td> + </tr> + + <tr valign="top"> + <td>Total</td> + <td>The total number of bytes processed by the filter + including errors. This is the maximum of the + <em>nbytes</em> argument or the return value. + </td></tr> + + <tr valign="top"> + <td>Errors</td> + <td>This field shows the number of bytes of the Total + column which can be attributed to errors.</td> + </tr> + + <tr valign="top"> + <td>User, System, Elapsed</td> + <td>These are the amount of user time, system time, and + elapsed time in seconds spent in the filter function. + Elapsed time is sensitive to system load. These times + may be zero on operating systems that don't support the + required operations.</td> + </tr> + + <tr valign="top"> + <td>Bandwidth</td> + <td>This is the filter bandwidth which is the total + number of bytes processed divided by elapsed time. + Since elapsed time is subject to system load the + bandwidth numbers cannot always be trusted. + Furthermore, the bandwidth includes bytes attributed to + errors which may significanly taint the value if the + function is able to detect errors without much + expense.</td> + </tr> + </tbody></table> + + <p> + </p> + <table> + <caption align="bottom"> + <b>Example: Filter Statistics</b> + </caption> + <tbody><tr> + <td> + <p><code></code></p><pre><code>H5Z: filter statistics accumulated ov= + er life of library: + Method Total Errors User System Elapsed Bandwidth + ------ ----- ------ ---- ------ ------- --------- + >deflate 160000 40000 0.62 0.74 1.33 117.5 kBs + <deflate 120000 0 0.11 0.00 0.12 1.000 MBs + </code></pre> + </td> + </tr> + </tbody></table> + + <hr> + + + <p><a name="fn1">Footnote 1:</a> Dataset chunks can be compressed + through the use of filters. Developers should be aware that + reading and rewriting compressed chunked data can result in holes + in an HDF5 file. In time, enough such holes can increase the + file size enough to impair application or library performance + when working with that file. See + <a href="https://support.hdfgroup.org/HDF5/doc1.6/Performance.html#Freespace"> + Freespace Management</a> + in the chapter + <a href="https://support.hdfgroup.org/HDF5/doc1.6/Performance.html"> + Performance Analysis and Issues</a>.</p> +</html> diff --git a/doxygen/examples/IOFlow.html b/doxygen/examples/IOFlow.html new file mode 100644 index 0000000..6b2c27e --- /dev/null +++ b/doxygen/examples/IOFlow.html @@ -0,0 +1,137 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<!-- saved from url=(0064)https://gamma.hdfgroup.org/papers/HISS/030821.IOFlow/IOFlow.html --> +<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <title>HDF5 Raw I/O Flow Notes</title> + + <meta name="author" content="Quincey Koziol"> +</head> + +<body text="#000000" bgcolor="#FFFFFF"> + +<style type="text/css"> +OL.loweralpha { list-style-type: lower-alpha } +OL.upperroman { list-style-type: upper-roman } +</style> + +<center><h1>HDF5 Raw I/O Flow Notes</h1></center> +<center><h3>Quincey Koziol<br> + koziol@ncsa.uiuc.edu<br> + August 20, 2003 +</h3></center> + +<ol class="upperroman"> + +<li><h3><u>Document's Audience:</u></h3> + +<ul> + <li>Current H5 library designers and knowledgable external developers.</li> +</ul> + +</li><li><h3><u>Background Reading:</u></h3> + +</li><li><h3><u>Introduction:</u></h3> + +<dl> + <dt><strong>What is this document about?</strong></dt> + <dd>This document attempts to supplement the flow charts describing + the flow of control for raw data I/O in the library. + </dd> <br> +</dl> + +</li><li><h3><u>Figures:</u></h3> +<p>The following figures provide the main information:</p> + <table> + <tr><td><img src="IOFlow.gif" alt="High-Level View of Writing Raw Data" style="height:50%;"></td></tr> + <tr><td><img src="IOFlow2.gif" alt="Perform Serial or Parallel I/O" style="height:50%;"></td></tr> + <tr><td><img src="IOFlow3.gif" alt="Gather/Convert/Scatter" style="height:50%;"></td></tr> + </table> + +</li><li><h3><u>Notes From Accompanying Figures:</u></h3> + +<p>This section provides notes to augment the information in the accompanying + figures. +</p> + +<ol> + <li><b>Validate Parameters</b> - Resolve any H5S_ALL parameters + for dataspace selections to actual dataspaces, allocate + conversion buffers, etc. + </li> + + <li><b>Space Allocated in File?</b> - Space may not have been allocated + in the file to store the dataset data, if "late allocation" was chosen + for the allocation time when the dataset was created. + </li> + + <li><b>Allocate & Fill Space</b> - These operations allocate both contiguous + and chunked dataset's space in the file. The chunked dataset space + allocation iterates through all the chunks in the file and allocates + both the B-tree information and the raw data in the file. Because of + the way filters work, fill-values are written out for chunked datasets + as they are allocated, instead of as a separate step. + In parallel + I/O, the chunked dataset allocation can potentially be time-consuming, + since all the raw data in the dataset is allocated from one process. + </li> + + <li><b>Datatype Conversion Needed?</b> - This currently is the deciding + factor between doing "direct I/O" (in serial or parallel) and needing + to perform gather/convert/scatter operations. I believe that MPI + is capable of performing a limited range of type conversions and if so, + we should add support to detect when they can be used. This will + allow more I/O operations to be performed collectively. + </li> + + <li><b>Collective I/O Requested/Allowed?</b> - A user has to both request + that collective I/O occur and also their I/O operation must meet the + requirements that the library sets for supporting collective parallel + I/O: + <ul> + <li>The dataspace must be scalar or simple (which is a no-op really, + since we don't support "complex" dataspaces in the library + currently). + </li> + <li>The selection must be regular. "all" selections + and hyperslab selections that were + made with only one call to H5Sselect_hyperslab() (i.e. not a + hyperslab selection that has been aggregated over multiple + selection calls) are regular. Supporting point and + irregular hyperslab selections are on the "to do" list. + </li> + <li>The dataset must be stored contiguously on disk (as shown in the + figure also). Supporting chunked dataset storage is also + on the "to do" list. + </li> + </ul> + </li> + + <li><b>Build "chunk map"</b> - This step still has some scalability issues + as it creates a data structure that is proportional to the number of + chunks which will be written to, which could potentially be very large. + Building the "chunk map" information incrementally is on the "to do" + list also. + </li> + + <li><b>Perform Chunked I/O</b> - As the figure shows, there is no support + for collective parallel I/O on chunked datasets currently. As noted + earlier, this is on the "to do" list. + </li> + + <li><b>Perform "Direct" Serial I/O</b> - "Direct" serial I/O writes data + from the application's buffer, without any intervening buffer or memory + copies. For maximum efficiency and performance, the elements in the + selections should be adjoining. + </li> + + <li><b>Perform Collective Parallel I/O</b> - This step also writes data + directly from an application buffer, but additionally uses collective + MPI I/O operations to combine the data from each process in the parallel + application in an efficient manner. + </li> +</ol> + +</li></ol> + + + +</body></html> diff --git a/doxygen/hdf5doxy_layout.xml b/doxygen/hdf5doxy_layout.xml index fc20aa1..6efa690 100644 --- a/doxygen/hdf5doxy_layout.xml +++ b/doxygen/hdf5doxy_layout.xml @@ -12,6 +12,7 @@ <tab type="user" url="@ref TN" title="Technical Notes" /> <tab type="user" url="@ref SPEC" title="Specifications" /> <tab type="user" url="@ref RFC" title="RFCs" /> + <tab type="user" url="@ref FTS" title="Full-Text Search" /> <tab type="user" url="@ref About" title="About" /> </navindex> diff --git a/doxygen/img/IOFlow.gif b/doxygen/img/IOFlow.gif Binary files differnew file mode 100644 index 0000000..3e79030 --- /dev/null +++ b/doxygen/img/IOFlow.gif diff --git a/doxygen/img/IOFlow2.gif b/doxygen/img/IOFlow2.gif Binary files differnew file mode 100644 index 0000000..c75ca79 --- /dev/null +++ b/doxygen/img/IOFlow2.gif diff --git a/doxygen/img/IOFlow3.gif b/doxygen/img/IOFlow3.gif Binary files differnew file mode 100644 index 0000000..316cd1e --- /dev/null +++ b/doxygen/img/IOFlow3.gif |
