diff options
Diffstat (limited to 'doxygen/examples/H5.format.2.0.html')
| -rw-r--r-- | doxygen/examples/H5.format.2.0.html | 14902 |
1 files changed, 14902 insertions, 0 deletions
diff --git a/doxygen/examples/H5.format.2.0.html b/doxygen/examples/H5.format.2.0.html new file mode 100644 index 0000000..3653489 --- /dev/null +++ b/doxygen/examples/H5.format.2.0.html @@ -0,0 +1,14902 @@ +<!DOCTYPE HTML> +<html> + <head> + <title> + HDF5 File Format Specification Version 2.0 + </title> + +<style> +h1 { display: block; + margin-top: 24px; + margin-bottom: 24px; + margin-left: 0px; + margin-right: 0px; + text-indent: 0px; + } + +h2 { display: block; + margin-top: 8x; + margin-bottom: 8px; + margin-left: 0px; + margin-right: 0px; + text-indent: 0px; + } +<!-- A horizontal rule (<hr />) should be placed on the line above +each h2 tag. The h2 tags are used on the main sections along with +the hr tags. --> + +h3 { display: block; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 0px; + margin-right: 0px; + text-indent: 0px; + } + +h4 { display: block; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 0px; + margin-right: 0px; + text-indent: 0px; + } + +p { display: block; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 0px; + margin-right: 0px; + text-indent: 0px; + } +<!-- +p.item { margin-left: 2em; + text-indent: -2em + } --> +<!-- p.item2 { margin-left: 2em; text-indent: 2em} --> + +table.format { border:solid; + border-collapse:collapse; + caption-side:top; + text-align:center; + width:80%; + } +table.format th { border:ridge; + padding:4px; + width:25%; + } +table.format td { border:ridge; + padding:4px; + } +table.format caption { font-weight:bold; + font-size:larger; + } + +table.note {border:none; + text-align:right; + width:80%; + } + +table.desc { border:solid; + border-collapse:collapse; + caption-size:top; + text-align:left; + width:80%; + } +table.desc tr { vertical-align:top; + } +table.desc th { border-style:ridge; + font-size:larger; + padding:4px; + <!-- text-decoration:underline; --> + } +table.desc td { border-style:ridge; + <!-- padding: 4px; --> + vertical-align:text-top; + } +table.desc caption { font-weight:bold; + font-size:larger; + } + +table.list { border:none; + width:100% + } +table.list tr { vertical-align:text-top; + } +table.list th { border:none; + text-decoration:underline; + vertical-align:text-top; + } +table.list td { border:none; + vertical-align:text-top; + } + +table.msgdesc { border:none; + text-align:left; + width: 80% + } +table.msgdesc tr { vertical-align:text-top; + border-spacing:0; + padding:0; } +table.msgdesc th { border:none; + text-decoration:underline; + vertical-align:text-top; } +table.msgdesc td { border:none; + vertical-align:text-top; + } + +table.list80 { border:none; + width:80% + } +table.list80 tr { vertical-align:text-top; + } +table.list80 th { border:none; + text-decoration:underline; + vertical-align:text-top; + } +table.list80 td { border:none; + vertical-align:text-top; + } + +table.glossary { border:none; + text-align:left; + width: 80% + } +table.glossary tr { vertical-align:text-top; + border-spacing:0; + padding:0; } +table.glossary th { border:none; + text-align:left; + text-decoration:underline; + vertical-align:text-top; } +table.glossary td { border:none; + text-align:left; + vertical-align:text-top; + } + +div { page-break-inside:avoid; + page-break-after:auto + } + +</style> + + <center> + <table border="0" width="90%"> + <tr> + <td valign="top"> + <ol type="I"> + <li><a href="#Intro">Introduction</a></li> + <font size="-1"> + <ol type="A"> + <li><a href="#ThisDocument">This Document</a></li> + <li><a href="#ChangesForHdf5_1_10">Changes for HDF5 1.10</a></li> + </ol> + </font> + + <li><a href="#FileMetaData">Disk Format: Level 0 - File Metadata</a></li> + <font size="-1"> + <ol type="A"> + <li><a href="#Superblock">Disk Format: Level 0A - Format Signature and Superblock</a></li> + <li><a href="#DriverInfo">Disk Format: Level 0B - File Driver Info</a></li> + <li><a href="#SuperblockExt">Disk Format: Level 0C - Superblock Extension</a></li> + </ol> + </font> + <li><a href="#FileInfra">Disk Format: Level 1 - File Infrastructure</a></li> + <font size="-1"> + <ol type="A"> + <li><a href="#Btrees">Disk Format: Level 1A - B-trees and B-tree + Nodes</a></li> + <ol type="1"> + <li><a href="#V1Btrees">Disk Format: Level 1A1 - Version 1 + B-trees (B-link Trees)</a></li> + <li><a href="#V2Btrees">Disk Format: Level 1A2 - Version 2 + B-trees</a></li> + </ol> + <li><a href="#SymbolTable">Disk Format: Level 1B - Group Symbol Table Nodes</a></li> + <li><a href="#SymbolTableEntry">Disk Format: Level 1C - Symbol Table Entry</a></li> + <li><a href="#LocalHeap">Disk Format: Level 1D - Local Heaps</a></li> + <li><a href="#GlobalHeap">Disk Format: Level 1E - Global Heap</a></li> + <li><a href="#FractalHeap">Disk Format: Level 1F - Fractal Heap</a></li> + <li><a href="#FreeSpaceManager">Disk Format: Level 1G - Free-space Manager</a></li> + <li><a href="#SOHMTable">Disk Format: Level 1H - Shared Object Header Message Table</a></li> + </ol> + </font> + <li><a href="#DataObject">Disk Format: Level 2 - Data Objects</a></li> + <font size="-1"> + <ol type="A"> + <li><a href="#ObjectHeader">Disk Format: Level 2A - Data Object Headers</a></li> + <ol type="1"> + <li><a href="#ObjectHeaderPrefix">Disk Format: Level 2A1 - Data Object Header Prefix</a></li> + <ol type="a"> + <li><a href="#V1ObjectHeaderPrefix">Version 1 Data Object Header Prefix</a></li> + <li><a href="#V2ObjectHeaderPrefix">Version 2 Data Object Header Prefix</a></li> + </ol> + <li><a href="#ObjectHeaderMessages">Disk Format: Level 2A2 - Data Object Header Messages</a></li> + <ol type="a"> + <li><a href="#NILMessage">The NIL Message</a></li> <!-- 0x0000 --> + <li><a href="#DataspaceMessage">The Dataspace Message</a></li> <!-- 0x0001 --> + <li><a href="#LinkInfoMessage">The Link Info Message</a></li> <!-- 0x0002 --> + </ol> + </ol> + </ol> + </font> + </ol> + </td> + + <td> </td> + + <td valign="top"> + <ol type="I" start="4"> + <li><a href="#DataObject">Disk Format: Level 2 - Data + Objects</a><font size="-1"><i> (Continued)</i></li> + <ol type="A"> + <li><a href="#ObjectHeader">Disk Format: Level 2A - Data Object + Headers</a><i> (Continued)</i></li> + <ol type="1" start="2"> + <li><a href="#ObjectHeaderMessages">Disk Format: Level 2A2 - + Data Object Header Messages</a><i> (Continued)</i></li> + <ol type="a" start="4"> + <li><a href="#DatatypeMessage">The Datatype Message</a></li> <!-- 0x0003 --> + <li><a href="#OldFillValueMessage">The Data Storage - + Fill Value (Old) Message</a></li> <!-- 0x0004 --> + <li><a href="#FillValueMessage">The Data Storage - + Fill Value Message</a></li> <!-- 0x0005 --> + <li><a href="#LinkMessage">The Link Message</a></li> <!-- 0x0006 --> + <li><a href="#ExternalFileListMessage">The Data Storage - + External Data Files Message</a></li> <!-- 0x0007 --> + <li><a href="#LayoutMessage">The Data Storage - + Layout Message</a></li> <!-- 0x0008 --> + <li><a href="#BogusMessage">The Bogus Message</a></li> <!-- 0x0009 --> + <li><a href="#GroupInfoMessage">The Group Info + Message</a></li> <!-- 0x000a --> + <li><a href="#FilterMessage">The Data Storage - + Filter Pipeline Message</a></li> <!-- 0x000b --> + <li><a href="#AttributeMessage">The Attribute + Message</a></li> <!-- 0x000c --> + <li><a href="#CommentMessage">The Object Comment + Message</a></li> <!-- 0x000d --> + <li><a href="#OldModificationTimeMessage">The Object + Modification Time (Old) Message</a></li> <!-- 0x000e --> + <li><a href="#SOHMTableMessage">The Shared Message + Table Message</a></li> <!-- 0x000f --> + <li><a href="#ContinuationMessage">The Object Header + Continuation Message</a></li> <!-- 0x0010 --> + <li><a href="#SymbolTableMessage">The Symbol + Table Message</a></li> <!-- 0x0011 --> + <li><a href="#ModificationTimeMessage">The Object + Modification Time Message</a></li> <!-- 0x0012 --> + <li><a href="#BtreeKValuesMessage">The B-tree + ‘K’ Values Message</a></li> <!-- 0x0013 --> + <li><a href="#DrvInfoMessage">The Driver Info + Message</a></li> <!-- 0x0014 --> + <li><a href="#AinfoMessage">The Attribute Info + Message</a></li> <!-- 0x0015 --> + <li><a href="#RefCountMessage">The Object Reference + Count Message</a></li> <!-- 0x0016 --> + <li><a href="#FsinfoMessage">The File Space Info + Message</a></li> <!-- 0x0018 --> + </ol> + </ol> + <li><a href="#DataStorage">Disk Format: Level 2B - Data Object Data Storage</a></li> + </ol> + </font> + <li><a href="#AppendixA">Appendix A: Definitions</a></li> + <li><a href="#AppendixB">Appendix B: File Memory Allocation Types</a></li> + </ol> +</td></tr> +</table> +</center> + + + +<br /> +<br /> +<hr /> +<a name="Intro"><h2>I. Introduction</h2></a> + + <table align="right" width="100"> + <tr><td> </td><td align="center"> + <hr /> + <img src="FF-IH_FileGroup.gif" alt="HDF5 Groups" hspace="15" vspace="15"> + </td><td> </td></tr> + <tr><td> </td><td align="center"> + <strong>Figure 1:</strong> Relationships among the HDF5 root group, other groups, and objects + <hr /> + </td><td> </td></tr> + + <tr><td> </td><td align="center"> + <img src="FF-IH_FileObject.gif" alt="HDF5 Objects" hspace="15" vspace="15"> + </td><td> </td></tr> + <tr><td> </td><td align="center"> + <strong>Figure 2:</strong> HDF5 objects -- datasets, datatypes, or dataspaces + <hr /> + </td><td> </td></tr> + </table> + + + <p>The format of an HDF5 file on disk encompasses several + key ideas of the HDF4 and AIO file formats as well as + addressing some shortcomings therein. The new format is + more self-describing than the HDF4 format and is more + uniformly applied to data objects in the file.</p> + + <p>An HDF5 file appears to the user as a directed graph. + The nodes of this graph are the higher-level HDF5 objects + that are exposed by the HDF5 APIs:</p> + + <ul> + <li>Groups</li> + <li>Datasets</li> + <li>Committed (formerly Named) datatypes</li> + </ul> + + <p>At the lowest level, as information is actually written to the disk, + an HDF5 file is made up of the following objects:</p> + <ul> + <li>A superblock</li> + <li>B-tree nodes</li> + <li>Heap blocks</li> + <li>Object headers</li> + <li>Object data</li> + <li>Free space</li> + </ul> + + <p>The HDF5 Library uses these low-level objects to represent the + higher-level objects that are then presented to the user or + to applications through the APIs. For instance, a group is an + object header that contains a message that points to a local + heap (for storing the links to objects in the group) and to a + B-tree (which indexes the links). A dataset is an object header + that contains messages that describe datatype, dataspace, layout, + filters, external files, fill value, and other elements with the + layout message pointing to either a raw data chunk or to a + B-tree that points to raw data chunks.</p> + + +<br /> +<a name="ThisDocument"><h3>I.A. This Document</h3></a> + + <p>This document describes the lower-level data objects; + the higher-level objects and their properties are described + in the <a href="UG/HDF5_Users_Guide-Responsive HTML5/index.html"><cite>HDF5 User’s Guide</cite></a>.</p> + + <p>Three levels of information comprise the file format. + Level 0 contains basic information for identifying and + defining information about the file. Level 1 information contains + the information about the pieces of a file shared by many objects + in the file (such as a B-trees and heaps). Level 2 is the rest + of the file and contains all of the data objects, with each object + partitioned into header information, also known as + <em>metadata</em>, and data.</p> + + <p>The sizes of various fields in the following layout tables are + determined by looking at the number of columns the field spans + in the table. There are three exceptions: (1) The size may be + overridden by specifying a size in parentheses, (2) the size of + addresses is determined by the <em>Size of Offsets</em> field + in the superblock and is indicated in this document with a + superscripted ‘O’, and (3) the size of length fields is determined + by the <em>Size of Lengths</em> field in the superblock and is + indicated in this document with a superscripted ‘L’.</p> + + <p>Values for all fields in this document should be treated as unsigned + integers, unless otherwise noted in the description of a field. + Additionally, all metadata fields are stored in little-endian byte + order. + </p> + + <p>All checksums used in the format are computed with the + <a href="http://www.burtleburtle.net/bob/hash/doobs.html">Jenkins’ + lookup3</a> algorithm. + </p> + + <p>Whenever a bit flag or field is mentioned for an entry, bits are + numbered from the lowest bit position in the entry. + </p> + + <p>Various tables in this document aligned with “This space inserted + only to align table nicely”. These entries in the table are just + to make the table presentation nicer and do not represent any values + or padding in the file. + </p> + + +<br /> +<a name="ChangesForHdf5_1_10"><h3>I.B. Changes for HDF5 1.10</h3></a> + + <p>As of October 2015, changes in the file format for HDF5 1.10 + have not yet been finalized.</p> + + + +<br /> +<br /> +<hr /> +<h2><a name="FileMetaData"> +II. Disk Format: Level 0 - File Metadata</a></h2> + +<br /> +<h3><a name="Superblock"> +II.A. Disk Format: Level 0A - Format Signature and Superblock</a></h3> + + <p>The superblock may begin at certain predefined offsets within + the HDF5 file, allowing a block of unspecified content for + users to place additional information at the beginning (and + end) of the HDF5 file without limiting the HDF5 Library’s + ability to manage the objects within the file itself. This + feature was designed to accommodate wrapping an HDF5 file in + another file format or adding descriptive information to an HDF5 + file without requiring the modification of the actual file’s + information. The superblock is located by searching for the + HDF5 format signature at byte offset 0, byte offset 512, and at + successive locations in the file, each a multiple of two of + the previous location; in other words, at these byte offsets: + 0, 512, 1024, 2048, and so on.</p> + + <p>The superblock is composed of the format signature, followed by a + superblock version number and information that is specific to each + version of the superblock. + Currently, there are three versions of the superblock format. + Version 0 is the default format, while version 1 is basically the same + as version 0 with additional information when a non-default B-tree ‘K’ + value is stored. Version 2 is the latest format, with some fields + eliminated or compressed and with superblock extension and checksum + support.</p> + + <p>Version 0 and 1 of the superblock are described below:</p> + + + <div align="center"> + <table class="format"> + <caption> + Superblock (Versions 0 and 1) + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Format Signature (8 bytes)<br /><br /></td> + </tr> + + <tr> + <td>Version # of Superblock</td> + <td>Version # of File’s Free Space Storage</td> + <td>Version # of Root Group Symbol Table Entry</td> + <td>Reserved (zero)</td> + </tr> + + <tr> + <td>Version # of Shared Header Message Format</td> + <td>Size of Offsets</td> + <td>Size of Lengths</td> + <td>Reserved (zero)</td> + </tr> + + <tr> + <td colspan="2">Group Leaf Node K</td> + <td colspan="2">Group Internal Node K</td> + </tr> + + <tr> + <td colspan="4">File Consistency Flags</td> + </tr> + + <tr> + <td colspan="2" style="border:dotted;">Indexed Storage Internal Node K<sup>1</sup></td> + <td colspan="2" style="border:dotted;">Reserved (zero)<sup>1</sup></td> + </tr> + + <tr> + <td colspan="4"><br />Base Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of File Free space Info<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />End of File Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Driver Information Block Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Root Group Symbol Table Entry</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are + of the size specified in “Size of Offsets.”) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with a ‘1’ in the above table are + new in version 1 of the superblock) + </td></tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Format Signature</p></td> + <td><p>This field contains a constant value and can be used to + quickly identify a file as being an HDF5 file. The + constant value is designed to allow easy identification of + an HDF5 file and to allow certain types of data corruption + to be detected. The file signature of an HDF5 file always + contains the following values:</p> + <center> + <table border align="center" cellpadding="4"> + <tr align="center"> + <td align="right">Decimal:</td> + <td width="8%">137</td> + <td width="8%">72</td> + <td width="8%">68</td> + <td width="8%">70</td> + <td width="8%">13</td> + <td width="8%">10</td> + <td width="8%">26</td> + <td width="8%">10</td> + </tr> + + <tr align="center"> + <td align="right">Hexadecimal:</td> + <td>89</td> + <td>48</td> + <td>44</td> + <td>46</td> + <td>0d</td> + <td>0a</td> + <td>1a</td> + <td>0a</td> + </tr> + + <tr align="center"> + <td align="right">ASCII C Notation:</td> + <td>\211</td> + <td>H</td> + <td>D</td> + <td>F</td> + <td>\r</td> + <td>\n</td> + <td>\032</td> + <td>\n</td> + </tr> + </table> + </center> + <p>This signature both identifies the file as an HDF5 file + and provides for immediate detection of common + file-transfer problems. The first two bytes distinguish + HDF5 files on systems that expect the first two bytes to + identify the file type uniquely. The first byte is + chosen as a non-ASCII value to reduce the probability + that a text file may be misrecognized as an HDF5 file; + also, it catches bad file transfers that clear bit + 7. Bytes two through four name the format. The CR-LF + sequence catches bad file transfers that alter newline + sequences. The control-Z character stops file display + under MS-DOS. The final line feed checks for the inverse + of the CR-LF translation problem. (This is a direct + descendent of the + <a href="http://www.libpng.org/pub/png/spec/iso/index-object.html#5PNG-file-signature">PNG</a> file + signature.)</p> + <p><em>This field is present in version 0+ of the superblock.</em> + </p></td> + </tr> + + <tr> + <td><p>Version Number of the Superblock</p></td> + <td><p>This value is used to determine the format of the + information in the superblock. When the format of the + information in the superblock is changed, the version number + is incremented to the next integer and can be used to + determine how the information in the superblock is + formatted.</p> + + <p>Values of 0, 1 and 2 are defined for this field. (The format + of version 2 is described below, not here) + </p> + + <p><em>This field is present in version 0+ of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Version Number of the File’s Free Space + Information</p></td> + <td> + <p>This value is used to determine the format of the + file’s free space information. + </p> + <p>The only value currently valid in this field is ‘0’, which + indicates that the file’s free space is as described + <a href="#FreeSpaceManager">below</a>. + </p> + + <p><em>This field is present in version 0 and 1 of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Version Number of the Root Group Symbol Table + Entry</p></td> + <td><p>This value is used to determine the format of the + information in the Root Group Symbol Table Entry. When the + format of the information in that field is changed, the + version number is incremented to the next integer and can be + used to determine how the information in the field + is formatted.</p> + <p>The only value currently valid in this field is ‘0’, + which indicates that the root group symbol table entry is + formatted as described <a href="#SymbolTableEntry">below</a>.</p> + <p><em>This field is present in version 0 and 1 of the + superblock.</em></p> + </td> + </tr> + + <tr> + <td><p>Version Number of the Shared Header Message Format</p></td> + <td><p>This value is used to determine the format of the + information in a shared object header message. Since the format + of the shared header messages differs from the other private + header messages, a version number is used to identify changes + in the format. + </p> + <p>The only value currently valid in this field is ‘0’, which + indicates that shared header messages are formatted as + described <a href="#ObjectHeaderMessages">below</a>. + </p> + + <p><em>This field is present in version 0 and 1 of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Size of Offsets</p></td> + <td><p>This value contains the number of bytes used to store + addresses in the file. The values for the addresses of + objects in the file are offsets relative to a base address, + usually the address of the superblock signature. This + allows a wrapper to be added after the file is created + without invalidating the internal offset locations. + </p> + + <p><em>This field is present in version 0+ of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Size of Lengths</p></td> + <td><p>This value contains the number of bytes used to store + the size of an object. + </p> + <p><em>This field is present in version 0+ of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Group Leaf Node K</p></td> + <td> + <p>Each leaf node of a group B-tree will have at + least this many entries but not more than twice this + many. If a group has a single leaf node then it + may have fewer entries. + </p> + <p>This value must be greater than zero. + </p> + <p>See the <a href="#Btrees">description</a> of B-trees below. + </p> + + <p><em>This field is present in version 0 and 1 of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Group Internal Node K</p></td> + <td> + <p>Each internal node of a group B-tree will have at + least this many entries but not more than twice this + many. If the group has only one internal + node then it might have fewer entries. + </p> + <p>This value must be greater than zero. + </p> + <p>See the <a href="#Btrees">description</a> of B-trees below. + </p> + + <p><em>This field is present in version 0 and 1 of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>File Consistency Flags</p></td> + <td> + <p>This value contains flags to indicate information + about the consistency of the information contained + within the file. Currently, the following bit flags are + defined: + <ul> + <li>Bit 0 set indicates that the file is opened for + write-access.</li> + <li>Bit 1 set indicates that the file has + been verified for consistency and is guaranteed to be + consistent with the format defined in this document.</li> + <li>Bits 2-31 are reserved for future use.</li> + </ul> + Bit 0 should be + set as the first action when a file is opened for write + access and should be cleared only as the final action + when closing a file. Bit 1 should be cleared during + normal access to a file and only set after the file’s + consistency is guaranteed by the library or a + consistency utility. + </p> + + <p><em>This field is present in version 0+ of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Indexed Storage Internal Node K</p></td> + <td> + <p>Each internal node of an indexed storage B-tree will have at + least this many entries but not more than twice this + many. If the index storage B-tree has only one internal + node then it might have fewer entries. + </p> + <p>This value must be greater than zero. + </p> + <p>See the <a href="#Btrees">description</a> of B-trees below. + </p> + + <p><em>This field is present in version 1 of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Base Address</p></td> + <td> + <p>This is the absolute file address of the first byte of + the HDF5 data within the file. The library currently + constrains this value to be the absolute file address + of the superblock itself when creating new files; + future versions of the library may provide greater + flexibility. When opening an existing file and this address does + not match the offset of the superblock, the library assumes + that the entire contents of the HDF5 file have been adjusted in + the file and adjusts the base address and end of file address to + reflect their new positions in the file. Unless otherwise noted, + all other file addresses are relative to this base + address. + </p> + + <p><em>This field is present in version 0+ of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Address of Global Free-space Index</p></td> + <td> + <p>The file’s free space is not persistent for version 0 and 1 of + the superblock. + Currently this field always contains the + <a href="#UndefinedAddress">undefined address</a>. + </p> + + <p><em>This field is present in version 0 and 1 of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>End of File Address</p></td> + <td> + <p>This is the absolute file address of the first byte past + the end of all HDF5 data. It is used to determine whether a + file has been accidently truncated and as an address where + file data allocation can occur if space from the free list is + not used. + </p> + + <p><em>This field is present in version 0+ of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Driver Information Block Address</p></td> + <td> + <p>This is the relative file address of the file driver + information block which contains driver-specific + information needed to reopen the file. If there is no + driver information block then this entry should be the + <a href="#UndefinedAddress">undefined address</a>. + </p> + + <p><em>This field is present in version 0 and 1 of the superblock.</em> + </p> + </td> + </tr> + + <tr> + <td><p>Root Group Symbol Table Entry</p></td> + <td> + <p>This is the <a href="#SymbolTableEntry">symbol table entry</a> + of the root group, which serves as the entry point into + the group graph for the file. + </p> + + <p><em>This field is present in version 0 and 1 of the superblock.</em> + </p> + </td> + </tr> + </table> + </div> + + <br /> + <p>Version 2 of the superblock is described below:</p> + + <div align="center"> + <table class="format"> + <caption> + Superblock (Version 2) + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Format Signature (8 bytes)<br /><br /></td> + </tr> + + <tr> + <td>Version # of Superblock</td> + <td>Size of Offsets</td> + <td>Size of Lengths</td> + <td>File Consistency Flags</td> + </tr> + + <tr> + <td colspan="4"><br />Base Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Superblock Extension Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />End of File Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Root Group Object Header Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Superblock Checksum</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are + of the size specified in “Size of Offsets.”) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Format Signature</p></td> + <td> + <p>This field is the same as described for versions 0 and 1 of the + superblock. + </p></td> + </tr> + + <tr> + <td><p>Version Number of the Superblock</p></td> + <td> + <p>This field has a value of 2 and has the same meaning as for + versions 0 and 1. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Offsets</p></td> + <td> + <p>This field is the same as described for versions 0 and 1 of the + superblock. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Lengths</p></td> + <td> + <p>This field is the same as described for versions 0 and 1 of the + superblock. + </p> + </td> + </tr> + + <tr> + <td><p>File Consistency Flags</p></td> + <td> + <p>This field is the same as described for versions 0 and 1 except + that it is smaller (the number of reserved bits has been reduced + from 30 to 6). + </p> + </td> + </tr> + + <tr> + <td><p>Base Address</p></td> + <td> + <p>This field is the same as described for versions 0 and 1 of the + superblock. + </p> + </td> + </tr> + + <tr> + <td><p>Superblock Extension Address</p></td> + <td> + <p>The field is the address of the object header for the + <a href="#SuperblockExt">superblock extension</a>. + If there is no extension then this entry should be the + <a href="#UndefinedAddress">undefined address</a>. + </p> + </td> + </tr> + + <tr> + <td><p>End of File Address</p></td> + <td> + <p>This field is the same as described for versions 0 and 1 of the + superblock. + </p> + </td> + </tr> + + <tr> + <td><p>Root Group Object Header Address</p></td> + <td> + <p>This is the address of + the <a href="#DataObject">root group object header</a>, + which serves as the entry point into the group graph for the file. + </p> + </td> + </tr> + + <tr> + <td><p>Superblock Checksum</p></td> + <td> + <p>The checksum for the superblock. + </p> + </td> + </tr> + + </table> + </div> + +<br /> +<h3><a name="DriverInfo"> +II.B. Disk Format: Level 0B - File Driver Info</a></h3> + + <p>The <b>driver information block</b> is an optional region of the + file which contains information needed by the file driver + to reopen a file. The format is described below:</p> + + + <div align="center"> + <table class="format"> + <caption> + Driver Information Block + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version</td> + <td colspan="3">Reserved</td> + </tr> + + <tr> + <td colspan="4">Driver Information Size</td> + </tr> + + <tr> + <td colspan="4"><br />Driver Identification (8 bytes)<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br /><br />Driver Information (<em>variable size</em>)<br /><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number of the Driver Information Block. + This document describes version 0. + </p> + </td> + </tr> + + <tr> + <td><p>Driver Information Size</p></td> + <td> + <p>The size in bytes of the <em>Driver Information</em> field. + </p> + </td> + </tr> + + <tr> + <td><p>Driver Identification</p></td> + <td> + <p>This is an eight-byte ASCII string without null + termination which identifies the driver and/or version number + of the Driver Information Block. The predefined driver encoded + in this field by the HDF5 Library is identified by the + letters <code>NCSA</code> followed by the first four characters of + the driver name. If the Driver Information block is not + the original version then the last letter(s) of the + identification will be replaced by a version number in + ASCII, starting with 0. + </p> + <p> + Identification for user-defined drivers is also eight-byte long. + It can be arbitrary but should be unique to avoid + the four character prefix “NCSA”. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Driver Information</p></td> + <td>Driver information is stored in a format defined by the + file driver (see description below).</td> + </tr> + </table> + </div> + + <br /> + The two drivers encoded in the <em>Driver Identification</em> field are as follows: + <ul> + <li> + Multi driver: + <p> + The identifier for this driver is “NCSAmulti”. + This driver provides a mechanism for segregating raw data and different types of metadata + into multiple files. + These files are viewed by the library as a single virtual HDF5 file with a single file address. + A maximum of 6 files will be created for the following data: + superblock, B-tree, raw data, global heap, local heap, and object header. + More than one type of data can be written to the same file. + </p></li> + <li> + Family driver + <p> + The identifier for this driver is “NCSAfami” and is encoded in this field for library version 1.8 and after. + This driver is designed for systems that do not support files larger than 2 gigabytes + by splitting the HDF5 file address space across several smaller files. + It does nothing to segregate metadata and raw data; + they are mixed in the address space just as they would be in a single contiguous file. + </p></li> + </ul> + <p>The format of the <em>Driver Information</em> field for the + above two drivers are described below:</p> + + <div align="center"> + <table class="format"> + <caption> + Multi Driver Information + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Member Mapping</td> + <td>Member Mapping</td> + <td>Member Mapping</td> + <td>Member Mapping</td> + </tr> + + <tr> + <td>Member Mapping</td> + <td>Member Mapping</td> + <td>Reserved</td> + <td>Reserved</td> + </tr> + + <tr> + <td colspan="4"><br />Address of Member File 1<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />End of Address for Member File 1<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of Member File 2<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />End of Address for Member File 2<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />... ...<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of Member File N<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />End of Address for Member File N<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Name of Member File 1 <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Name of Member File 2 <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />... ...<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Name of Member File N <em>(variable size)</em><br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Member Mapping</p></td> + <td><p>These fields are integer values from 1 to 6 + indicating how the data can be mapped to or merged with another type of + data. + <table class="list"> + <tr> + <th width="20%" align="center">Member Mapping</th> + <th width="80%" align="left">Description</th> + </tr> + <tr> + <td align="center">1</td> + <td>The superblock data.</td> + </tr> + <tr> + <td align="center">2</td> + <td>The B-tree data.</td> + </tr> + <tr> + <td align="center">3</td> + <td>The raw data.</td> + </tr> + <tr> + <td align="center">4</td> + <td>The global heap data.</td> + </tr> + <tr> + <td align="center">5</td> + <td>The local heap data.</td> + </tr> + <tr> + <td align="center">6</td> + <td>The object header data.</td> + </tr> + </table></p> + <p>For example, if the third field has the value 3 and all the rest have the + value 1, it means there are two files: one for raw data, and one for superblock, + B-tree, global heap, local heap, and object header.</p> + </td> + </tr> + + <tr> + <td><p>Reserved</p></td> + <td><p>These fields are reserved and should always be zero.</p></td> + </tr> + + <tr> + <td><p>Address of Member File N</p></td> + <td><p>This field Specifies the virtual address at which the member file starts.</p> + <p>N is the number of member files.</p> + </td> + </tr> + + <tr> + <td><p>End of Address for Member File N</p></td> + <td><p>This field is the end of the allocated address for the member file. + </p></td> + </tr> + + <tr> + <td><p>Name of Member File N</p></td> + <td><p>This field is the null-terminated name of the member file and + its length should be multiples of 8 bytes. + Additional bytes will be padded with <em>NULL</em>s. The default naming + convention is <em>%s-X.h5</em>, where <em>X</em> is one of the letters + <em>s</em> (for superblock), <em>b</em> (for B-tree), <em>r</em> (for raw data), + <em>g</em> (for global heap), <em>l</em> (for local heap), and <em>o</em> (for + object header). The name of the whole HDF5 file will substitute the <em>%s</em> + in the string. + </p> + </td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Family Driver Information + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="8"><br />Size of Member File<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Size of Member File</p></td> + <td><p>This field is the size of the member file in the family of files.</p></td> + </tr> + </table> + </div> + +<br /> +<h3><a name="SuperblockExt"> +II.C. Disk Format: Level 0C - Superblock Extension</a></h3> + + <p>The <em>superblock extension</em> is used to store superblock metadata + which is either optional, or added after the version of the superblock + was defined. Superblock extensions may only exist when version 2+ of + superblock is used. A superblock extension is an object header which may + hold the following messages:</p> + <ul> + <li> + <a href="#SOHMTableMessage">Shared Message Table message</a> containing + information to locate the master table of shared object header message + indices.</li> + <li> + <a href="#BtreeKValuesMessage">B-tree ‘K’ Values message</a> containing + non-default B-tree ‘K’ values.</li> + <li> + <a href="#DrvInfoMessage">Driver Info message</a> containing information + needed by the file driver in order to reopen a file. + See also the + <a href="#DriverInfo">“Disk Format: Level 0B - File Driver + Info”</a> section above.</li> + <li> + <a href="#FsinfoMessage">File Space Info message</a> containing + information about file space handling in the file.</li> + </ul> + + + +<br /> +<br /> +<hr /> +<h2><a name="FileInfra"> +III. Disk Format: Level 1 - File Infrastructure</a></h2> + +<br /> +<h3><a name="Btrees"> +III.A. Disk Format: Level 1A - B-trees and B-tree Nodes</a></h3> + + <p>B-trees allow flexible storage for objects which tend to grow + in ways that cause the object to be stored discontiguously. B-trees + are described in various algorithms books including “Introduction to + Algorithms” by Thomas H. Cormen, Charles E. Leiserson, and Ronald + L. Rivest. B-trees are used in several places in the HDF5 file format, + when an index is needed for another data structure.</p> + + <p>The version 1 B-tree structure described below is the original index + structure, but are limited by some bugs in our implementation (mainly in + how they handle deleting records). The version 1 B-trees are being phased + out in favor of the version 2 B-trees described below, although both + types of structures may be found in the same file, depending on + application settings when creating the file.</p> + +<br /> +<h4><a name="V1Btrees"> +III.A.1. Disk Format: Level 1A1 - Version 1 B-trees (B-link Trees)</a></h4> + + <p>Version 1 B-trees in HDF5 files an implementation of the B-link tree, + in which the sibling nodes at a particular level in the tree are stored + in a doubly-linked list, is described in the “Efficient Locking for + Concurrent Operations on B-trees” paper by Phillip Lehman and S. Bing Yao + as published in the <cite>ACM Transactions on Database Systems</cite>, + Vol. 6, No. 4, December 1981.</p> + + <p>The B-link trees implemented by the file format contain one more + key than the number of children. In other words, each child + pointer out of a B-tree node has a left key and a right key. + The pointers out of internal nodes point to sub-trees while + the pointers out of leaf nodes point to symbol nodes and + raw data chunks. + Aside from that difference, internal nodes and leaf nodes + are identical.</p> + + <div align="center"> + <table class="format"> + <caption> + B-link Tree Nodes + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Node Type</td> + <td>Node Level</td> + <td colspan="2">Entries Used</td> + </tr> + + <tr> + <td colspan="4"><br />Address of Left Sibling<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of Right Sibling<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Key 0 (variable size)</td> + </tr> + + <tr> + <td colspan="4"><br />Address of Child 0<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Key 1 (variable size)</td> + </tr> + + <tr> + <td colspan="4"><br />Address of Child 1<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">Key 2<em>K</em> (variable size)</td> + </tr> + + <tr> + <td colspan="4"><br />Address of Child 2<em>K</em><sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Key 2<em>K</em>+1 (variable size)</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>TREE</code>” is + used to indicate the + beginning of a B-link tree node. This gives file + consistency checking utilities a better chance of + reconstructing a damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Node Type</p></td> + <td> + <p>Each B-link tree points to a particular type of data. + This field indicates the type of data as well as + implying the maximum degree <em>K</em> of the tree and + the size of each Key field. + + + <table class="list"> + <tr> + <th width="20%" align="center">Node Type</th> + <th width="80%" align="left">Description</th> + </tr> + <tr> + <td align="center">0</td> + <td>This tree points to group nodes.</td> + </tr> + <tr> + <td align="center">1</td> + <td>This tree points to raw data chunk nodes.</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Node Level</p></td> + <td> + <p>The node level indicates the level at which this node + appears in the tree (leaf nodes are at level zero). Not + only does the level indicate whether child pointers + point to sub-trees or to data, but it can also be used + to help file consistency checking utilities reconstruct + damaged trees. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Entries Used</p></td> + <td> + <p>This determines the number of children to which this + node points. All nodes of a particular type of tree + have the same maximum degree, but most nodes will point + to less than that number of children. The valid child + pointers and keys appear at the beginning of the node + and the unused pointers and keys appear at the end of + the node. The unused pointers and keys have undefined + values. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Address of Left Sibling</p></td> + <td> + <p>This is the relative file address of the left sibling of + the current node. If the current + node is the left-most node at this level then this field + is the <a href="#UndefinedAddress">undefined address</a>. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Address of Right Sibling</p></td> + <td> + <p>This is the relative file address of the right sibling of + the current node. If the current + node is the right-most node at this level then this + field is the <a href="#UndefinedAddress">undefined address</a>. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Keys and Child Pointers</p></td> + <td> + <p>Each tree has 2<em>K</em>+1 keys with 2<em>K</em> + child pointers interleaved between the keys. The number + of keys and child pointers actually containing valid + values is determined by the node’s <em>Entries Used</em> field. + If that field is <em>N</em> then the B-link tree contains + <em>N</em> child pointers and <em>N</em>+1 keys. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Key</p></td> + <td> + <p>The format and size of the key values is determined by + the type of data to which this tree points. The keys are + ordered and are boundaries for the contents of the child + pointer; that is, the key values represented by child + <em>N</em> fall between Key <em>N</em> and Key + <em>N</em>+1. Whether the interval is open or closed on + each end is determined by the type of data to which the + tree points. + </p> + + <p> + The format of the key depends on the node type. + For nodes of node type 0 (group nodes), the key is formatted as + follows: + + <table class="list"> + <tr> + <td width="20%">A single field of <i>Size of Lengths</i> + bytes:</td> + <td width="80%">Indicates the byte offset into the local heap + for the first object name in the subtree which + that key describes. + </td> + </tr> + </table> + </p> + + + <p> + For nodes of node type 1 (chunked raw data nodes), the key is + formatted as follows: + + <table class="list"> + <tr> + <td width="20%">Bytes 1-4:</td> + <td width="80%">Size of chunk in bytes.</td> + </tr> + <tr> + <td>Bytes 4-8:</td> + <td>Filter mask, a 32-bit bit field indicating which + filters have been skipped for this chunk. Each filter + has an index number in the pipeline (starting at 0, with + the first filter to apply) and if that filter is skipped, + the bit corresponding to its index is set.</td> + </tr> + <tr> + <td>(<em>D + 1</em>) 64-bit fields:</td> + <td>The offset of the + chunk within the dataset where <i>D</i> is the number + of dimensions of the dataset, and the last value is the + offset within the dataset’s datatype and should always be + zero. For example, if + a chunk in a 3-dimensional dataset begins at the + position <code>[5,5,5]</code>, there will be three + such 64-bit values, each with the value of + <code>5</code>, followed by a <code>0</code> value.</td> + </tr> + </table> + </p> + + </td> + </tr> + + <tr valign="top"> + <td><p>Child Pointer</p></td> + <td> + <p>The tree node contains file addresses of subtrees or + data depending on the node level. Nodes at Level 0 point + to data addresses, either raw data chunks or group nodes. + Nodes at non-zero levels point to other nodes of the + same B-tree. + </p> + <p>For raw data chunk nodes, the child pointer is the address + of a single raw data chunk. For group nodes, the child pointer + points to a <a href="#SymbolTable">symbol table</a>, which contains + information for multiple symbol table entries. + </p> + </td> + </tr> + </table> + </div> + + <p> + Conceptually, each B-tree node looks like this:</p> + <center> + <table> + <tr valign="top" align="center"> + <td>key[0]</td><td> </td> + <td>child[0]</td><td> </td> + <td>key[1]</td><td> </td> + <td>child[1]</td><td> </td> + <td>key[2]</td><td> </td> + <td>...</td><td> </td> + <td>...</td><td> </td> + <td>key[<i>N</i>-1]</td><td> </td> + <td>child[<i>N</i>-1]</td><td> </td> + <td>key[<i>N</i>]</td> + </tr> + </table> + </center> + <br /> + + where child[<i>i</i>] is a pointer to a sub-tree (at a level + above Level 0) or to data (at Level 0). + Each key[<i>i</i>] describes an <i>item</i> stored by the B-tree + (a chunk or an object of a group node). The range of values + represented by child[<i>i</i>] is indicated by key[<i>i</i>] + and key[<i>i</i>+1]. + + + <p>The following question must next be answered: + “Is the value described by key[<i>i</i>] contained in + child[<i>i</i>-1] or in child[<i>i</i>]?” + The answer depends on the type of tree. + In trees for groups (node type 0) the object described by + key[<i>i</i>] is the greatest object contained in + child[<i>i</i>-1] while in chunk trees (node type 1) the + chunk described by key[<i>i</i>] is the least chunk in + child[<i>i</i>].</p> + + <p>That means that key[0] for group trees is sometimes unused; + it points to offset zero in the heap, which is always the + empty string and compares as “less-than” any valid object name.</p> + + <p>And key[<i>N</i>] for chunk trees is sometimes unused; + it contains a chunk offset which compares as “greater-than” + any other chunk offset and has a chunk byte size of zero + to indicate that it is not actually allocated.</p> + +<br /> +<h4><a name="V2Btrees"> +III.A.2. Disk Format: Level 1A2 - Version 2 B-trees</a></h4> + + <p>Version 2 B-trees are “traditional” B-trees, with one major difference. + Instead of just using a simple pointer (or address in the file) to a + child of an internal node, the pointer to the child node contains two + additional pieces of information: the number of records in the child + node itself, and the total number of records in the child node and + all its descendants. Storing this additional information allows fast + array-like indexing to locate the n<sup>th</sup> record in the B-tree.</p> + + <p>The entry into a version 2 B-tree is a header which contains global + information about the structure of the B-tree. The <em>root node + address</em> + field in the header points to the B-tree root node, which is either an + internal or leaf node, depending on the value in the header’s + <em>depth</em> field. An internal node consists of records plus + pointers to further leaf or internal nodes in the tree. A leaf node + consists of solely of records. The format of the records depends on + the B-tree type (stored in the header).</p> + + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree Header + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + + <tr> + <td colspan="4">Signature</td> + </tr> + <tr> + <td>Version</td> + <td>Type</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4">Node Size</td> + </tr> + <tr> + <td colspan="2">Record Size</td> + <td colspan="2">Depth</td> + </tr> + <tr> + <td>Split Percent</td> + <td>Merge Percent</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4"><br />Root Node Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="2">Number of Records in Root Node</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4"><br />Total Number of Records in B-tree<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>BTHD</code>” is + used to indicate the header of a version 2 B-link tree node. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number for this B-tree header. This document + describes version 0. + </p> + </td> + </tr> + + <tr> + <td><p>Type</p></td> + <td> + <p>This field indicates the type of B-tree: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + <tr> + <td align="center">0</td> + <td>A “testing” B-tree, this value should <em>not</em> be + used for storing records in actual HDF5 files. + </td> + </tr> + <tr> + <td align="center">1</td> + <td>This B-tree is used for indexing indirectly accessed, + non-filtered ‘huge’ fractal heap objects. + </td> + </tr> + <tr> + <td align="center">2</td> + <td>This B-tree is used for indexing indirectly accessed, + filtered ‘huge’ fractal heap objects. + </td> + </tr> + <tr> + <td align="center">3</td> + <td>This B-tree is used for indexing directly accessed, + non-filtered ‘huge’ fractal heap objects. + </td> + </tr> + <tr> + <td align="center">4</td> + <td>This B-tree is used for indexing directly accessed, + filtered ‘huge’ fractal heap objects. + </td> + </tr> + <tr> + <td align="center">5</td> + <td>This B-tree is used for indexing the ‘name’ field for + links in indexed groups. + </td> + </tr> + <tr> + <td align="center">6</td> + <td>This B-tree is used for indexing the ‘creation order’ + field for links in indexed groups. + </td> + </tr> + <tr> + <td align="center">7</td> + <td>This B-tree is used for indexing shared object header + messages. + </td> + </tr> + <tr> + <td align="center">8</td> + <td>This B-tree is used for indexing the ‘name’ field for + indexed attributes. + </td> + </tr> + <tr> + <td align="center">9</td> + <td>This B-tree is used for indexing the ‘creation order’ + field for indexed attributes. + </td> + </tr> + </table></p> + <p>The format of records for each type is described below.</p> + </td> + </tr> + + <tr valign="top"> + <td><p>Node Size</p></td> + <td> + <p>This is the size in bytes of all B-tree nodes. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Record Size</p></td> + <td> + <p>This field is the size in bytes of the B-tree record. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Depth</p></td> + <td> + <p>This is the depth of the B-tree. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Split Percent</p></td> + <td> + <p>The percent full that a node needs to increase above before it + is split. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Merge Percent</p></td> + <td> + <p>The percent full that a node needs to be decrease below before it + is split. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Root Node Address</p></td> + <td> + <p>This is the address of the root B-tree node. A B-tree with + no records will have the <a href="#UndefinedAddress">undefined + address</a> in this field. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Number of Records in Root Node</p></td> + <td> + <p>This is the number of records in the root node. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Total Number of Records in B-tree</p></td> + <td> + <p>This is the total number of records in the entire B-tree. + </p> + </td> + </tr> + + <tr valign="top"> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the B-tree header. + </p> + </td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree Internal Node + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + <tr> + <td>Version</td> + <td>Type</td> + <td colspan="2">Records 0, 1, 2...N-1 <em>(variable size)</em></td> + </tr> + <tr> + <td colspan="4"><br />Child Node Pointer 0<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Number of Records N<sub>0</sub> for Child Node 0 <em>(variable size)</em></td> + </tr> + <tr> + <td colspan="4"><br />Total Number of Records for Child Node 0 <em>(optional, variable size)</em></td> + </tr> + <tr> + <td colspan="4"><br />Child Node Pointer 1<sup>O</sup><br /><br /></td> + </tr> + <td colspan="4"><br />Number of Records N<sub>1</sub> for Child Node 1 <em>(variable size)</em></td> + </tr> + <tr> + <td colspan="4"><br />Total Number of Records for Child Node 1 <em>(optional, variable size)</em></td> + </tr> + <tr> + <td colspan="4">...</td> + </tr> + <tr> + <td colspan="4"><br />Child Node Pointer N<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Number of Records N<sub>n</sub> for Child Node N <em>(variable size)</em></td> + </tr> + <tr> + <td colspan="4"><br />Total Number of Records for Child Node N <em>(optional, variable size)</em></td> + </tr> + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + </div> + + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>BTIN</code>” is + used to indicate the internal node of a B-link tree. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number for this B-tree internal node. + This document describes version 0. + </p> + </td> + </tr> + + <tr> + <td><p>Type</p></td> + <td> + <p>This field is the type of the B-tree node. It should always + be the same as the B-tree type in the header. + </p> + </td> + </tr> + + <tr> + <td><p>Records</p></td> + <td> + <p>The size of this field is determined by the number of records + for this node and the record size (from the header). The format + of records depends on the type of B-tree. + </p> + </td> + </tr> + + <tr> + <td><p>Child Node Pointer</p></td> + <td> + <p>This field is the address of the child node pointed to by the + internal node. + </p> + </td> + </tr> + + <tr> + <td><p>Number of Records in Child Node</p></td> + <td> + <p>This is the number of records in the child node pointed to by + the corresponding <em>Node Pointer</em>. + </p> + <p>The number of bytes used to store this field is determined by + the maximum possible number of records able to be stored in the + child node. + </p> + <p> + The maximum number of records in a child node is computed + in the following way: + + <ul> + <li>Subtract the fixed size overhead for + the child node (for example, its signature, version, + checksum, and so on and <em>one</em> pointer triplet + of information for the child node (because there is one + more pointer triplet than records in each internal node)) + from the size of nodes for the B-tree. </li> + <li>Divide that result by the size of a record plus the + pointer triplet of information stored to reach each + child node from this node. + </ul> + + </p> + <p> + Note that leaf nodes do not encode any + child pointer triplets, so the maximum number of records in a + leaf node is just the node size minus the leaf node overhead, + divided by the record size. + </p> + <p> + Also note that the first level of internal nodes above the + leaf nodes do not encode the <em>Total Number of Records in Child + Node</em> value in the child pointer triplets (since it is the + same as the <em>Number of Records in Child Node</em>), so the + maximum number of records in these nodes is computed with the + equation above, but using (<em>Child Pointer</em>, <em>Number of + Records in Child Node</em>) pairs instead of triplets. + </p> + <p> + The number of + bytes used to encode this field is the least number of bytes + required to encode the maximum number of records in a child + node value for the child nodes below this level + in the B-tree. + </p> + <p> + For example, if the maximum number of child records is + 123, one byte will be used to encode these values in this + node; if the maximum number of child records is + 20000, two bytes will be used to encode these values in this + node; and so on. The maximum number of bytes used to + encode these values is 8 (in other words, an unsigned + 64-bit integer). + </p> + </td> + </tr> + + <tr> + <td><p>Total Number of Records in Child Node</p></td> + <td> + <p>This is the total number of records for the node pointed to by + the corresponding <em>Node Pointer</em> and all its children. + This field exists only in nodes whose depth in the B-tree node + is greater than 1 (in other words, the “twig” + internal nodes, just above leaf nodes, do not store this + field in their child node pointers). + </p> + <p>The number of bytes used to store this field is determined by + the maximum possible number of records able to be stored in the + child node and its descendants. + </p> + <p> + The maximum possible number of records able to be stored in a + child node and its descendants is computed iteratively, in the + following way: The maximum number of records in a leaf node + is computed, then that value is used to compute the maximum + possible number of records in the first level of internal nodes + above the leaf nodes. Multiplying these two values together + determines the maximum possible number of records in child node + pointers for the level of nodes two levels above leaf nodes. + This process is continued up to any level in the B-tree. + </p> + <p> + The number of bytes used to encode this value is computed in + the same way as for the <em>Number of Records in Child Node</em> + field. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for this node. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree Leaf Node + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + <tr> + <td>Version</td> + <td>Type</td> + <td colspan="2">Record 0, 1, 2...N-1 <em>(variable size)</em></td> + </tr> + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>BTLF</code>“ is + used to indicate the leaf node of a version 2 B-link tree. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number for this B-tree leaf node. + This document describes version 0. + </p> + </td> + </tr> + + <tr> + <td><p>Type</p></td> + <td> + <p>This field is the type of the B-tree node. It should always + be the same as the B-tree type in the header. + </p> + </td> + </tr> + + <tr> + <td><p>Records</p></td> + <td> + <p>The size of this field is determined by the number of records + for this node and the record size (from the header). The format + of records depends on the type of B-tree. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for this node. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <p>The record layout for each stored (in other words, non-testing) + B-tree type is as follows:</p> + + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 1 Record Layout - Indirectly Accessed, Non-Filtered, + ‘Huge’ Fractal Heap Objects + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Huge Object Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Huge Object Length<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Huge Object ID<sup>L</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Huge Object Address</p></td> + <td> + <p>The address of the huge object in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Huge Object Length</p></td> + <td> + <p>The length of the huge object in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Huge Object ID</p></td> + <td> + <p>The heap ID for the huge object. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 2 Record Layout - Indirectly Accessed, Filtered, + ‘Huge’ Fractal Heap Objects + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Filtered Huge Object Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Filtered Huge Object Length<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">Filter Mask</td> + </tr> + <tr> + <td colspan="4"><br />Filtered Huge Object Memory Size<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Huge Object ID<sup>L</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Filtered Huge Object Address</p></td> + <td> + <p>The address of the filtered huge object in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Filtered Huge Object Length</p></td> + <td> + <p>The length of the filtered huge object in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Filter Mask</p></td> + <td> + <p>A 32-bit bit field indicating which filters have been skipped for + this chunk. Each filter has an index number in the pipeline + (starting at 0, with the first filter to apply) and if that + filter is skipped, the bit corresponding to its index is set. + </p> + </td> + </tr> + + <tr> + <td><p>Filtered Huge Object Memory Size</p></td> + <td> + <p>The size of the de-filtered huge object in memory. + </p> + </td> + </tr> + + <tr> + <td><p>Huge Object ID</p></td> + <td> + <p>The heap ID for the huge object. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 3 Record Layout - Directly Accessed, Non-Filtered, + ‘Huge’ Fractal Heap Objects + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Huge Object Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Huge Object Length<sup>L</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Huge Object Address</p></td> + <td> + <p>The address of the huge object in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Huge Object Length</p></td> + <td> + <p>The length of the huge object in the file. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 4 Record Layout - Directly Accessed, Filtered, + ‘Huge’ Fractal Heap Objects + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Filtered Huge Object Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Filtered Huge Object Length<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">Filter Mask</td> + </tr> + <tr> + <td colspan="4"><br />Filtered Huge Object Memory Size<sup>L</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Filtered Huge Object Address</p></td> + <td> + <p>The address of the filtered huge object in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Filtered Huge Object Length</p></td> + <td> + <p>The length of the filtered huge object in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Filter Mask</p></td> + <td> + <p>A 32-bit bit field indicating which filters have been skipped for + this chunk. Each filter has an index number in the pipeline + (starting at 0, with the first filter to apply) and if that + filter is skipped, the bit corresponding to its index is set. + </p> + </td> + </tr> + + <tr> + <td><p>Filtered Huge Object Memory Size</p></td> + <td> + <p>The size of the de-filtered huge object in memory. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 5 Record Layout - Link Name for Indexed Group + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Hash of Name</td> + </tr> + <tr> + <td colspan="4">ID <em>(bytes 1-4)</em></td> + </tr> + + <tr> + <td colspan="3">ID <em>(bytes 5-7)</em></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Hash</p></td> + <td> + <p>This field is hash value of the name for the link. The hash + value is the Jenkins’ lookup3 checksum algorithm applied to + the link’s name. + </p> + </td> + </tr> + + <tr> + <td><p>ID</p></td> + <td> + <p>This is a 7-byte sequence of bytes and is the heap ID for the + link record in the group’s fractal heap.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 6 Record Layout - Creation Order for Indexed Group + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Creation Order <em>(8 bytes)</em><br /><br /></td> + </tr> + <tr> + <td colspan="4">ID <em>(bytes 1-4)</em></td> + </tr> + <tr> + <td colspan="3">ID <em>(bytes 5-7)</em></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Creation Order</p></td> + <td> + <p>This field is the creation order value for the link. + </p> + </td> + </tr> + + <tr> + <td><p>ID</p></td> + <td> + <p>This is a 7-byte sequence of bytes and is the heap ID for the + link record in the group’s fractal heap.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages (Sub-Type 0 - Message in Heap) + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan>Message Location</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4">Hash</td> + </tr> + <tr> + <td colspan="4">Reference Count</td> + </tr> + <tr> + <td colspan="4"><br />Heap ID <em>(8 bytes)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Message Location</p></td> + <td> + <p>This field Indicates the location where the message is stored: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + <tr> + <td align="center">0</td> + <td>Shared message is stored in shared message index heap. + </td> + </tr> + <tr> + <td align="center">1</td> + <td>Shared message is stored in object header. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Hash</p></td> + <td> + <p>This field is hash value of the shared message. The hash + value is the Jenkins’ lookup3 checksum algorithm applied to + the shared message.</p> + </td> + </tr> + + <tr> + <td><p>Reference Count</p></td> + <td> + <p>The number of objects which reference this message.</p> + </td> + </tr> + + <tr> + <td><p>Heap ID</p></td> + <td> + <p>This is an 8-byte sequence of bytes and is the heap ID for the + shared message in the shared message index’s fractal heap.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 7 Record Layout - Shared Object Header Messages (Sub-Type 1 - Message in Object Header) + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan>Message Location</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4">Hash</td> + </tr> + <tr> + <td>Reserved (zero)</td> + <td>Message Type</td> + <td colspan="2">Object Header Index</td> + </tr> + <tr> + <td colspan="4"><br />Object Header Address<sup>O</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Message Location</p></td> + <td> + <p>This field Indicates the location where the message is stored: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + <tr> + <td align="center">0</td> + <td>Shared message is stored in shared message index heap. + </td> + </tr> + <tr> + <td align="center">1</td> + <td>Shared message is stored in object header. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Hash</p></td> + <td> + <p>This field is hash value of the shared message. The hash + value is the Jenkins’ lookup3 checksum algorithm applied to + the shared message.</p> + </td> + </tr> + + <tr> + <td><p>Message Type</p></td> + <td> + <p>The object header message type of the shared message.</p> + </td> + </tr> + + <tr> + <td><p>Object Header Index</p></td> + <td> + <p>This field indicates that the shared message is the n<sup>th</sup> message + of its type in the specified object header.</p> + </td> + </tr> + + <tr> + <td><p>Object Header Address</p></td> + <td> + <p>The address of the object header containing the shared message.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 8 Record Layout - Attribute Name for Indexed Attributes + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Heap ID <em>(8 bytes)</em><br /><br /></td> + </tr> + <tr> + <td colspan>Message Flags</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4">Creation Order</td> + </tr> + <tr> + <td colspan="4">Hash of Name</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Heap ID</p></td> + <td> + <p>This is an 8-byte sequence of bytes and is the heap ID for the + attribute in the object’s attribute fractal heap.</p> + </td> + </tr> + + <tr> + <td><p>Message Flags</p></td> + <td><p>The object header message flags for the attribute message.</p> + </td> + </tr> + + <tr> + <td><p>Creation Order</p></td> + <td> + <p>This field is the creation order value for the attribute. + </p> + </td> + </tr> + + <tr> + <td><p>Hash</p></td> + <td> + <p>This field is hash value of the name for the attribute. The hash + value is the Jenkins’ lookup3 checksum algorithm applied to + the attribute’s name. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Version 2 B-tree, Type 9 Record Layout- Creation Order for Indexed Attributes + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Heap ID <em>(8 bytes)</em><br /><br /></td> + </tr> + <tr> + <td colspan>Message Flags</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4">Creation Order</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Heap ID</p></td> + <td> + <p>This is an 8-byte sequence of bytes and is the heap ID for the + attribute in the object’s attribute fractal heap.</p> + </td> + </tr> + + <tr> + <td><p>Message Flags</p></td> + <td> + <p>The object header message flags for the attribute message.</p> + </td> + </tr> + + <tr> + <td><p>Creation Order</p></td> + <td> + <p>This field is the creation order value for the attribute. + </p> + </td> + </tr> + + </table> + </div> + + +<br /> +<h3><a name="SymbolTable"> +III.B. Disk Format: Level 1B - Group Symbol Table Nodes</a></h3> + + <p>A group is an object internal to the file that allows + arbitrary nesting of objects within the file (including other groups). + A group maps a set of link names in the group to a set of relative + file addresses of objects in the file. Certain metadata for an object to + which the group points can be cached in the group’s symbol table entry in + addition to being in the object’s header.</p> + + <p>An HDF5 object name space can be stored hierarchically by + partitioning the name into components and storing each + component as a link in a group. The link for a + non-ultimate component points to the group containing + the next component. The link for the last + component points to the object being named.</p> + + <p>One implementation of a group is a collection of symbol table nodes + indexed by a B-link tree. Each symbol table node contains entries + for one or more links. If an attempt is made to add a link to an already + full symbol table node containing 2<em>K</em> entries, then the node is + split and one node contains <em>K</em> symbols and the other contains + <em>K</em>+1 symbols.</p> + + <div align="center"> + <table class="format"> + <caption> + Symbol Table Node (A Leaf of a B-link tree) + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version Number</td> + <td>Reserved (zero)</td> + <td colspan="2">Number of Symbols</td> + </tr> + + <tr> + <td colspan="4"><br /><br />Group Entries<br /><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>SNOD</code>” is + used to indicate the + beginning of a symbol table node. This gives file + consistency checking utilities a better chance of + reconstructing a damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version Number</p></td> + <td> + <p>The version number for the symbol table node. This + document describes version 1. (There is no version ‘0’ + of the symbol table node) + </p> + </td> + </tr> + + <tr> + <td><p>Number of Entries</p></td> + <td> + <p>Although all symbol table nodes have the same length, + most contain fewer than the maximum possible number of + link entries. This field indicates how many entries + contain valid data. The valid entries are packed at the + beginning of the symbol table node while the remaining + entries contain undefined values. + </p> + </td> + </tr> + + <tr> + <td><p>Symbol Table Entries</p></td> + <td> + <p>Each link has an entry in the symbol table node. + The format of the entry is described below. + There are 2<em>K</em> entries in each group node, where + <em>K</em> is the “Group Leaf Node K” value from the + <a href="#Superblock">superblock</a>. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h3><a name="SymbolTableEntry"> +III.C. Disk Format: Level 1C - Symbol Table Entry </a></h3> + + <p>Each symbol table entry in a symbol table node is designed + to allow for very fast browsing of stored objects. + Toward that design goal, the symbol table entries + include space for caching certain constant metadata from the + object header.</p> + + <div align="center"> + <table class="format"> + <caption> + Symbol Table Entry + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Link Name Offset<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Object Header Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Cache Type</td> + </tr> + + <tr> + <td colspan="4">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br /><br />Scratch-pad Space (16 bytes)<br /><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Link Name Offset</p></td> + <td> + <p>This is the byte offset into the group’s local + heap for the name of the link. The name is null + terminated. + </p> + </td> + </tr> + + <tr> + <td><p>Object Header Address</p></td> + <td> + <p>Every object has an object header which serves as a + permanent location for the object’s metadata. In addition + to appearing in the object header, some of the object’s metadata + can be cached in the scratch-pad space. + </p> + </td> + </tr> + + <tr> + <td><p>Cache Type</p></td> + <td> + <p>The cache type is determined from the object header. + It also determines the format for the scratch-pad space: + + <table class="list"> + <tr> + <th width="20%" align="center">Type</th> + <th width="80%" align="left">Description</th> + </tr> + <tr> + <td align="center">0</td> + <td>No data is cached by the group entry. This + is guaranteed to be the case when an object header + has a link count greater than one. + </td> + </tr> + <tr> + <td align="center">1</td> + <td>Group object header metadata is cached in the + scratch-pad space. This implies that the symbol table + entry refers to another group. + </td> + </tr> + <tr> + <td align="center">2</td> + <td>The entry is a symbolic link. The first four bytes + of the scratch-pad space are the offset into the local + heap for the link value. The object header address + will be undefined. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Reserved</p></td> + <td> + <p>These four bytes are present so that the scratch-pad + space is aligned on an eight-byte boundary. They are + always set to zero. + </p> + </td> + </tr> + + <tr> + <td><p>Scratch-pad Space</p></td> + <td> + <p>This space is used for different purposes, depending + on the value of the Cache Type field. Any metadata + about an object represented in the scratch-pad + space is duplicated in the object header for that + object. + </p> + <p> + Furthermore, no data is cached in the group + entry scratch-pad space if the object header for + the object has a link count greater than one. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h4>Format of the Scratch-pad Space</h4> + + <p>The symbol table entry scratch-pad space is formatted + according to the value in the Cache Type field.</p> + + <p>If the Cache Type field contains the value zero + <code>(0)</code> then no information is + stored in the scratch-pad space.</p> + + <p>If the Cache Type field contains the value one + <code>(1)</code>, then the scratch-pad space + contains cached metadata for another object header + in the following format:</p> + + <div align="center"> + <table class="format"> + <caption> + Object Header Scratch-pad Format + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Address of B-tree<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of Name Heap<sup>O</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Address of B-tree</p></td> + <td> + <p>This is the file address for the root of the + group’s B-tree. + </p> + </td> + </tr> + + <tr> + <td><p>Address of Name Heap</p></td> + <td> + <p>This is the file address for the group’s local + heap, in which are stored the group’s symbol names. + </p> + </td> + </tr> + </table> + </div> + + + <br /> + <p>If the Cache Type field contains the value two + <code>(2)</code>, then the scratch-pad space + contains cached metadata for a symbolic link + in the following format:</p> + + <div align="center"> + <table class="format"> + <caption> + Symbolic Link Scratch-pad Format + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Offset to Link Value</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Offset to Link Value</p></td> + <td> + <p>The value of a symbolic link (that is, the name of the + thing to which it points) is stored in the local heap. + This field is the 4-byte offset into the local heap for + the start of the link value, which is null terminated. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h3><a name="LocalHeap"> +III.D. Disk Format: Level 1D - Local Heaps</a></h3> + + <p>A local heap is a collection of small pieces of data that are particular + to a single object in the HDF5 file. Objects can be + inserted and removed from the heap at any time. + The address of a heap does not change once the heap is created. + For example, a group stores addresses of objects in symbol table nodes + with the names of links stored in the group’s local heap. + </p> + + <div align="center"> + <table class="format"> + <caption> + Local Heap + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br />Data Segment Size<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Offset to Head of Free-list<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of Data Segment<sup>O</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>HEAP</code>” + is used to indicate the + beginning of a heap. This gives file consistency + checking utilities a better chance of reconstructing a + damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>Each local heap has its own version number so that new + heaps can be added to old files. This document + describes version zero (0) of the local heap. + </p> + </td> + </tr> + + <tr> + <td><p>Data Segment Size</p></td> + <td> + <p>The total amount of disk memory allocated for the heap + data. This may be larger than the amount of space + required by the objects stored in the heap. The extra + unused space in the heap holds a linked list of free blocks. + </p> + </td> + </tr> + + <tr> + <td><p>Offset to Head of Free-list</p></td> + <td> + <p>This is the offset within the heap data segment of the + first free block (or the + <a href="#UndefinedAddress">undefined address</a> if there is no + free block). The free block contains “Size of Lengths” bytes that + are the offset of the next free block (or the + value ‘1’ if this is the + last free block) followed by “Size of Lengths” bytes that store + the size of this free block. The size of the free block includes + the space used to store the offset of the next free block and + the size of the current block, making the minimum size of a free + block 2 * “Size of Lengths”. + </p> + </td> + </tr> + + <tr> + <td><p>Address of Data Segment</p></td> + <td> + <p>The data segment originally starts immediately after + the heap header, but if the data segment must grow as a + result of adding more objects, then the data segment may + be relocated, in its entirety, to another part of the + file. + </p> + </td> + </tr> + </table> + </div> + + <p>Objects within a local heap should be aligned on an 8-byte boundary.</p> + +<br /> +<h3><a name="GlobalHeap"> +III.E. Disk Format: Level 1E - Global Heap</a></h3> + + <p>Each HDF5 file has a global heap which stores various types of + information which is typically shared between datasets. The + global heap was designed to satisfy these goals:</p> + + <ol type="A"> + <li>Repeated access to a heap object must be efficient without + resulting in repeated file I/O requests. Since global heap + objects will typically be shared among several datasets, it is + probable that the object will be accessed repeatedly.</li> + <li>Collections of related global heap objects should result in + fewer and larger I/O requests. For instance, a dataset of + object references will have a global heap object for each + reference. Reading the entire set of object references + should result in a few large I/O requests instead of one small + I/O request for each reference.</li> + <li>It should be possible to remove objects from the global heap + and the resulting file hole should be eligible to be reclaimed + for other uses.</li> + </ol> + + + <p>The implementation of the heap makes use of the memory management + already available at the file level and combines that with a new + object called a <em>collection</em> to achieve goal B. The global heap + is the set of all collections. Each global heap object belongs to + exactly one collection and each collection contains one or more global + heap objects. For the purposes of disk I/O and caching, a collection is + treated as an atomic object, addressing goal A. + </p> + + <p>When a global heap object is deleted from a collection (which occurs + when its reference count falls to zero), objects located after the + deleted object in the collection are packed down toward the beginning + of the collection and the collection’s global heap object 0 is created + (if possible) or its size is increased to account for the recently + freed space. There are no gaps between objects in each collection, + with the possible exception of the final space in the collection, if + it is not large enough to hold the header for the collection’s global + heap object 0. These features address goal C. + </p> + + <p>The HDF5 Library creates global heap collections as needed, so there may + be multiple collections throughout the file. The set of all of them is + abstractly called the “global heap”, although they do not actually link + to each other, and there is no global place in the file where you can + discover all of the collections. The collections are found simply by + finding a reference to one through another object in the file. For + example, data of variable-length datatype elements is stored in the + global heap and is accessed via a global heap ID. The format for + global heap IDs is described at the end of this section. + </p> + + <div align="center"> + <table class="format"> + <caption> + A Global Heap Collection + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br />Collection Size<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Global Heap Object 1<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Global Heap Object 2<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />...<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Global Heap Object <em>N</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Global Heap Object 0 (free space)<br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>GCOL</code>” + is used to indicate the + beginning of a collection. This gives file consistency + checking utilities a better chance of reconstructing a + damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>Each collection has its own version number so that new + collections can be added to old files. This document + describes version one (1) of the collections (there is no + version zero (0)). + </p> + </td> + </tr> + + <tr> + <td><p>Collection Size</p></td> + <td> + <p>This is the size in bytes of the entire collection + including this field. The default (and minimum) + collection size is 4096 bytes which is a typical file + system block size. This allows for 127 16-byte heap + objects plus their overhead (the collection header of 16 bytes + and the 16 bytes of information about each heap object). + </p> + </td> + </tr> + + <tr> + <td><p>Global Heap Object 1 through <em>N</em></p></td> + <td> + <p>The objects are stored in any order with no + intervening unused space. + </p> + </td> + </tr> + + <tr> + <td><p>Global Heap Object 0</p></td> + <td> + <p>Global Heap Object 0 (zero), when present, represents the free + space in the collection. Free space always appears at the end of + the collection. If the free space is too small to store the header + for Object 0 (described below) then the header is implied and the + collection contains no free space. + </p> + </td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Global Heap Object + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="2">Heap Object Index</td> + <td colspan="2">Reference Count</td> + </tr> + + <tr> + <td colspan="4">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br />Object Size<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Object Data<br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Heap Object Index</p></td> + <td> + <p>Each object has a unique identification number within a + collection. The identification numbers are chosen so that + new objects have the smallest value possible with the + exception that the identifier <code>0</code> always refers to the + object which represents all free space within the + collection. + </p> + </td> + </tr> + + <tr> + <td><p>Reference Count</p></td> + <td> + <p>All heap objects have a reference count field. An + object which is referenced from some other part of the + file will have a positive reference count. The reference + count for Object 0 is always zero. + </p> + </td> + </tr> + + <tr> + <td><p>Reserved</p></td> + <td> + <p>Zero padding to align next field on an 8-byte boundary. + </p> + </td> + </tr> + + <tr> + <td><p>Object Size</p></td> + <td> + <p>This is the size of the object data stored for the object. + The actual storage space allocated for the object data is rounded + up to a multiple of eight. + </p> + </td> + </tr> + + <tr> + <td><p>Object Data</p></td> + <td> + <p>The object data is treated as a one-dimensional array + of bytes to be interpreted by the caller. + </p> + </td> + </tr> + </table> + + </div> + + <br /> + <p> + The format for the ID used to locate an object in the global heap is + described here:</p> + + <div align="center"> + <table class="format"> + <caption> + Global Heap ID + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Collection Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Object Index</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Collection Address</p></td> + <td> + <p>This field is the address of the global heap collection + where the data object is stored. + </p> + </td> + </tr> + + <tr> + <td><p>ID</p></td> + <td> + <p>This field is the index of the data object within the + global heap collection. + </p> + </td> + </tr> + + </table> + </div> + + +<br /> +<h3><a name="FractalHeap"> +III.F. Disk Format: Level 1F - Fractal Heap</a></h3> + + <p> + Each fractal heap consists of a header and zero or more direct and + indirect blocks (described below). The header contains general + information as well as + initialization parameters for the doubling table. The <em>Root + Block Address</em> in the header points to the first direct or + indirect block in the heap. + </p> + + <p> + Fractal heaps are based on a data structure called a <em>doubling + table</em>. A doubling table provides a mechanism for quickly + extending an array-like data structure that minimizes the number of + empty blocks in the heap, while retaining very fast lookup of any + element within the array. More information on fractal heaps and + doubling tables can be found in the RFC + “<a href="Supplements/FractalHeap/PrivateHeap.pdf">Private + Heaps in HDF5</a>.” + </p> + + <p> + The fractal heap implements the doubling table structure with + indirect and direct blocks. + Indirect blocks in the heap do not actually contain data for + objects in the heap, their “size” is abstract - + they represent the indexing structure for locating the + direct blocks in the doubling table. + Direct blocks + contain the actual data for objects stored in the heap. + </p> + + <p> + All indirect blocks have a constant number of block entries in each + row, called the <em>width</em> of the doubling table (stored in + the heap header). + + The number + of rows for each indirect block in the heap is determined by the + size of the block that the indirect block represents in the + doubling table (calculation of this is shown below) and is + constant, except for the “root” + indirect block, which expands and shrinks its number of rows as + needed. + </p> + + <p> + Blocks in the first <em>two</em> rows of an indirect block + are <em>Starting Block Size</em> number of bytes in size, + and the blocks in each subsequent row are twice the size of + the blocks in the previous row. In other words, blocks in + the third row are twice the <em>Starting Block Size</em>, + blocks in the fourth row are four times the + <em>Starting Block Size</em>, and so on. Entries for + blocks up to the <em>Maximum Direct Block Size</em> point to + direct blocks, and entries for blocks greater than that size + point to further indirect blocks (which have their own + entries for direct and indirect blocks). + </p> + + <p> + The number of rows of blocks, <em>nrows</em>, in an + indirect block of size <em>iblock_size</em> is given by the + following expression: + <br /> <br /> + <em>nrows</em> = (log<sub>2</sub>(<em>iblock_size</em>) - + log<sub>2</sub>(<em><Starting Block Size></em> * + <em><Width></em>)) + 1 + </p> + + <p> + The maximum number of rows of direct blocks, <em>max_dblock_rows</em>, + in any indirect block of a fractal heap is given by the + following expression: + <br /> <br /> + <em>max_dblock_rows</em> = + (log<sub>2</sub>(<em><Max. Direct Block Size></em>) - + log<sub>2</sub>(<em><Starting Block Size></em>)) + 2 + </p> + + <p> + Using the computed values for <em>nrows</em> and + <em>max_dblock_rows</em>, along with the <em>Width</em> of the + doubling table, the number of direct and indirect block entries + (<em>K</em> and <em>N</em> in the indirect block description, below) + in an indirect block can be computed: + <br /> <br /> + <em>K</em> = MIN(<em>nrows</em>, <em>max_dblock_rows</em>) * + <em>Width</em> + + <br /> <br /> + If <em>nrows</em> is less than or equal to <em>max_dblock_rows</em>, + <em>N</em> is 0. Otherwise, <em>N</em> is simply computed: + <br /> <br /> + <em>N</em> = <em>K</em> - (<em>max_dblock_rows</em> * + <em>Width</em>) + </p> + + <p> + The size indirect blocks on disk is determined by the number + of rows in the indirect block (computed above). The size of direct + blocks on disk is exactly the size of the block in the doubling + table. + </p> + + <div align="center"> + <table class="format"> + <caption> + Fractal Heap Header + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="2">Heap ID Length</td> + <td colspan="2">I/O Filters’ Encoded Length</td> + </tr> + + <tr> + <td>Flags</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Maximum Size of Managed Objects</td> + </tr> + + <tr> + <td colspan="4"><br />Next Huge Object ID<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />v2 B-tree Address of Huge Objects<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Amount of Free Space in Managed Blocks<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of Managed Block Free Space Manager<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Amount of Managed Space in Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Amount of Allocated Managed Space in Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Offset of Direct Block Allocation Iterator in Managed Space<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Number of Managed Objects in Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Size of Huge Objects in Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Number of Huge Objects in Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Size of Tiny Objects in Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Number of Tiny Objects in Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="2">Table Width</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Starting Block Size<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Maximum Direct Block Size<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="2">Maximum Heap Size</td> + <td colspan="2">Starting # of Rows in Root Indirect Block</td> + </tr> + + <tr> + <td colspan="4"><br />Address of Root Block<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="2">Current # of Rows in Root Indirect Block</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Size of Filtered Root Direct Block <em>(optional)</em><sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">I/O Filter Mask<em> (optional)</em></td> + </tr> + + <tr> + <td colspan="4">I/O Filter Information<em> (optional, variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="40%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>FRHP</code>” + is used to indicate the + beginning of a fractal heap header. This gives file consistency + checking utilities a better chance of reconstructing a + damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This document describes version 0.</p> + </td> + </tr> + + <tr> + <td><p>Heap ID Length</p></td> + <td> + <p>This is the length in bytes of heap object IDs for this heap.</p> + </td> + </tr> + + <tr> + <td><p>I/O Filters’ Encoded Length</p></td> + <td> + <p>This is the size in bytes of the encoded <em>I/O Filter Information</em>. + </p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td> + <p>This field is the heap status flag and is a bit field + indicating additional information about the fractal heap. + <table class="list"> + <tr> + <th width="20%" align="center">Bit(s)</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, the ID value to use for huge object has wrapped + around. If the value for the <em>Next Huge Object ID</em> + has wrapped around, each new huge object inserted into the + heap will require a search for an ID value. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>If set, the direct blocks in the heap are checksummed. + </td> + </tr> + <tr> + <td align="center"><code>2-7</code></td> + <td>Reserved</td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Maximum Size of Managed Objects</p></td> + <td> + <p>This is the maximum size of managed objects allowed in the heap. + Objects greater than this this are ‘huge’ objects and will be + stored in the file directly, rather than in a direct block for + the heap. + </p> + </td> + </tr> + + <tr> + <td><p>Next Huge Object ID</p></td> + <td> + <p>This is the next ID value to use for a huge object in the heap. + </p> + </td> + </tr> + + <tr> + <td><p>v2 B-tree Address of Huge Objects</p></td> + <td> + <p>This is the address of the <a href="#V2Btrees">v2 B-tree</a> + used to track huge objects in the heap. The type of records + stored in the <em>v2 B-tree</em> will + be determined by whether the address & length of a huge object + can fit into a heap ID (if yes, it is a “directly” accessed + huge object) and whether there is a filter used on objects + in the heap. + </p> + </td> + </tr> + + <tr> + <td><p>Amount of Free Space in Managed Blocks</p></td> + <td> + <p>This is the total amount of free space in managed direct blocks + (in bytes). + </p> + </td> + </tr> + + <tr> + <td><p>Address of Managed Block Free Space Manager</p></td> + <td> + <p>This is the address of the + <em><a href="#FreeSpaceManager">Free-space Manager</a></em> for + managed blocks. + </p> + </td> + </tr> + + <tr> + <td><p>Amount of Managed Space in Heap</p></td> + <td> + <p>This is the total amount of managed space in the heap (in bytes), + essentially the upper bound of the heap’s linear address space. + </p> + </td> + </tr> + + <tr> + <td><p>Amount of Allocated Managed Space in Heap</p></td> + <td> + <p>This is the total amount of managed space (in bytes) actually + allocated in + the heap. This can be less than the <em>Amount of Managed Space + in Heap</em> field, if some direct blocks in the heap’s linear + address space are not allocated. + </p> + </td> + </tr> + + <tr> + <td><p>Offset of Direct Block Allocation Iterator in Managed Space</p></td> + <td> + <p>This is the linear heap offset where the next direct + block should be allocated at (in bytes). This may be less than + the <em>Amount of Managed Space in Heap</em> value because the + heap’s address space is increased by a “row” of direct blocks + at a time, rather than by single direct block increments. + </p> + </td> + </tr> + + <tr> + <td><p>Number of Managed Objects in Heap</p></td> + <td> + <p>This is the number of managed objects in the heap. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Huge Objects in Heap</p></td> + <td> + <p>This is the total size of huge objects in the heap (in bytes). + </p> + </td> + </tr> + + <tr> + <td><p>Number of Huge Objects in Heap</p></td> + <td> + <p>This is the number of huge objects in the heap. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Tiny Objects in Heap</p></td> + <td> + <p>This is the total size of tiny objects that are packed in heap + IDs (in bytes). + </p> + </td> + </tr> + + <tr> + <td><p>Number of Tiny Objects in Heap</p></td> + <td> + <p>This is the number of tiny objects that are packed in heap IDs. + </p> + </td> + </tr> + + <tr> + <td><p>Table Width</p></td> + <td> + <p>This is the number of columns in the doubling table for managed + blocks. This value must be a power of two. + </p> + </td> + </tr> + + <tr> + <td><p>Starting Block Size</p></td> + <td> + <p>This is the starting block size to use in the doubling table for + managed blocks (in bytes). This value must be a power of two. + </p> + </td> + </tr> + + <tr> + <td><p>Maximum Direct Block Size</p></td> + <td> + <p>This is the maximum size allowed for a managed direct block. + Objects inserted into the heap that are larger than this value + (less the # of bytes of direct block prefix/suffix) + are stored as ‘huge’ objects. This value must be a power of + two. + </p> + </td> + </tr> + + <tr> + <td><p>Maximum Heap Size</p></td> + <td> + <p>This is the maximum size of the heap’s linear address space for + managed objects (in bytes). The value stored is the log2 of + the actual value, that is: the # of bits of the address space. + ‘Huge’ and ‘tiny’ objects are not counted in this value, since + they do not store objects in the linear address space of the + heap. + </p> + </td> + </tr> + + <tr> + <td><p>Starting # of Rows in Root Indirect Block</p></td> + <td> + <p>This is the starting number of rows for the root indirect block. + A value of 0 indicates that the root indirect block will have + the maximum number of rows needed to address the heap’s <em>Maximum + Heap Size</em>. + </p> + </td> + </tr> + + <tr> + <td><p>Address of Root Block</p></td> + <td> + <p>This is the address of the root block for the heap. It can + be the <a href="#UndefinedAddress">undefined address</a> if + there is no data in the heap. It either points to a direct + block (if the <em>Current # of Rows in the Root Indirect Block</em> + value is 0), or an indirect block. + </p> + </td> + </tr> + + <tr> + <td><p>Current # of Rows in Root Indirect Block</p></td> + <td> + <p>This is the current number of rows in the root indirect block. + A value of 0 indicates that <em>Address of Root Block</em> + points to direct block instead of indirect block. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Filtered Root Direct Block</p></td> + <td> + <p>This is the size of the root direct block, if filters are + applied to heap objects (in bytes). This field is only + stored in the header if the <em>I/O Filters’ Encoded Length</em> + is greater than 0. + </p> + </td> + </tr> + + <tr> + <td><p>I/O Filter Mask</p></td> + <td> + <p>This is the filter mask for the root direct block, if filters + are applied to heap objects. This mask has the same format as + that used for the filter mask in chunked raw data records in a + <a href="#V1Btrees">v1 B-tree</a>. + This field is only + stored in the header if the <em>I/O Filters’ Encoded Length</em> + is greater than 0. + </p> + </td> + </tr> + + <tr> + <td><p>I/O Filter Information</p></td> + <td> + <p>This is the I/O filter information encoding direct blocks and + huge objects, if filters are applied to heap objects. This + field is encoded as a <a href="#FilterMessage">Filter Pipeline</a> + message. + The size of this field is determined by <em>I/O Filters’ + Encoded Length</em>. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the header.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Fractal Heap Direct Block + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Heap Header Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Block Offset <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Checksum <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Object Data <em>(variable size)</em><br /><br /></td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>FHDB</code>” + is used to indicate the + beginning of a fractal heap direct block. This gives file consistency + checking utilities a better chance of reconstructing a + damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This document describes version 0.</p> + </td> + </tr> + + <tr> + <td><p>Heap Header Address</p></td> + <td> + <p>This is the address for the fractal heap header that this + block belongs to. This field is principally used for file + integrity checking. + </p> + </td> + </tr> + + <tr> + <td><p>Block Offset</p></td> + <td> + <p>This is the offset of the block within the fractal heap’s + address space (in bytes). The number of bytes used to encode + this field is the <em>Maximum Heap Size</em> (in the heap’s + header) divided by 8 and rounded up to the next highest integer, + for values that are not a multiple of 8. This value is + principally used for file integrity checking. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the direct block.</p> + <p>This field is only present if bit 1 of <em>Flags</em> in the + heap’s header is set.</p> + </td> + </tr> + + <tr> + <td><p>Object Data</p></td> + <td> + <p>This section of the direct block stores the actual data for + objects in the heap. The size of this section is determined by + the direct block’s size minus the size of the other fields + stored in the direct block (for example, the <em>Signature</em>, + <em>Version</em>, and others including the <em>Checksum</em> if it is + present). + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Fractal Heap Indirect Block + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Heap Header Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Block Offset <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Child Direct Block #0 Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Size of Filtered Direct Block #0 <em>(optional)</em> <sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">Filter Mask for Direct Block #0 <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Child Direct Block #1 Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Size of Filtered Direct Block #1 <em>(optional)</em><sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">Filter Mask for Direct Block #1 <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4"><br />Child Direct Block #K-1 Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Size of Filtered Direct Block #K-1 <em>(optional)</em><sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">Filter Mask for Direct Block #K-1 <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Child Indirect Block #0 Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Child Indirect Block #1 Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4"><br />Child Indirect Block #N-1 Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>FHIB</code>” is used to + indicate the beginning of a fractal heap indirect block. This + gives file consistency checking utilities a better chance of + reconstructing a damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This document describes version 0.</p> + </td> + </tr> + + <tr> + <td><p>Heap Header Address</p></td> + <td> + <p>This is the address for the fractal heap header that this + block belongs to. This field is principally used for file + integrity checking. + </p> + </td> + </tr> + + <tr> + <td><p>Block Offset</p></td> + <td> + <p>This is the offset of the block within the fractal heap’s + address space (in bytes). The number of bytes used to encode + this field is the <em>Maximum Heap Size</em> (in the heap’s + header) divided by 8 and rounded up to the next highest integer, + for values that are not a multiple of 8. This value is + principally used for file integrity checking. + </p> + </td> + </tr> + + <tr> + <td><p>Child Direct Block #K Address</p></td> + <td> + <p>This field is the address of the child direct block. + The size of the [uncompressed] direct block can be computed by + its offset in the heap’s linear address space. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Filtered Direct Block #K</p></td> + <td> + <p>This is the size of the child direct block after passing through + the I/O filters defined for this heap (in bytes). If no I/O + filters are present for this heap, this field is not present. + </p> + </td> + </tr> + <tr> + <td><p>Filter Mask for Direct Block #K</p></td> + <td> + <p>This is the I/O filter mask for the filtered direct block. + This mask has the same format as that used for the filter mask + in chunked raw data records in a <a href="#V1Btrees">v1 B-tree</a>. + If no I/O filters are present for this heap, this field is not + present. + </p> + </td> + </tr> + + <tr> + <td><p>Child Indirect Block #N Address</p></td> + <td> + <p>This field is the address of the child indirect block. + The size of the indirect block can be computed by + its offset in the heap’s linear address space. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the indirect block.</p> + </td> + </tr> + + </table> + + </div> + + <br /> + <p>An object in the fractal heap is identified by means of a fractal heap ID, + which encodes information to locate the object in the heap. + Currently, the fractal heap stores an object in one of three ways, + depending on the object’s size:</p> + + <div align="center"> + <table class="list80"> + <tr> + <th width="20%">Type</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center">Tiny</td> + <td> + <p>When an object is small enough to be encoded in the heap ID, the + object’s data is embedded in the fractal heap ID itself. There are + 2 sub-types for this type of object: normal and extended. The + sub-type for tiny heap IDs depends on whether the heap ID is large + enough to store objects greater than 16 bytes or not. If the + heap ID length is 18 bytes or smaller, the ‘normal’ tiny heap ID + form is used. If the heap ID length is greater than 18 bytes in + length, the “extented” form is used. See format description below + for both sub-types. + </p> + </td> + </tr> + + <tr> + <td align="center">Huge</td> + <td> + <p>When the size of an object is larger than <em>Maximum Size of + Managed Objects</em> in the <em>Fractal Heap Header</em>, the + object’s data is stored on its own in the file and the object + is tracked/indexed via a version 2 B-tree. All huge objects + for a particular fractal heap use the same v2 B-tree. All huge + objects for a particular fractal heap use the same format for + their huge object IDs. + </p> + + <p>Depending on whether the IDs for a heap are large enough to hold + the object’s retrieval information and whether I/O pipeline filters + are applied to the heap’s objects, 4 sub-types are derived for + huge object IDs for this heap:</p> + + <div align="center"> + <table class="list"> + <tr> + <th align="left" width="35%">Sub-type</th> + <th align="left">Description</th> + </tr> + + <tr> + <td align="left">Directly accessed, non-filtered</td> + <td> + <p>The object’s address and length are embedded in the + fractal heap ID itself and the object is directly accessed + from them. This allows the object to be accessed without + resorting to the B-tree. + </p> + </td> + </tr> + + <tr> + <td align="left">Directly accessed, filtered</td> + <td> + <p>The filtered object’s address, length, filter mask and + de-filtered size are embedded in the fractal heap ID itself + and the object is accessed directly with them. This allows + the object to be accessed without resorting to the B-tree. + </p> + </td> + </tr> + + <tr> + <td align="left">Indirectly accessed, non-filtered</td> + <td> + <p>The object is located by using a B-tree key embedded in + the fractal heap ID to retrieve the address and length from + the version 2 B-tree for huge objects. Then, the address + and length are used to access the object. + </p> + </td> + </tr> + + <tr> + <td align="left">Indirectly accessed, filtered</td> + <td> + <p>The object is located by using a B-tree key embedded in + the fractal heap ID to retrieve the filtered object’s + address, length, filter mask and de-filtered size from the + version 2 B-tree for huge objects. Then, this information + is used to access the object. + </p> + </td> + </tr> + </table> + </div> + + </td> + </tr> + + <tr> + <td align="center">Managed</td> + <td> + <p>When the size of an object does not meet the above two + conditions, the object is stored and managed via the direct and + indirect blocks based on the doubling table. + </p> + </td> + </tr> + </table> + </div> + + + <p>The specific format for each type of heap ID is described below: + </p> + + <div align="center"> + <table class="format"> + <caption>Fractal Heap ID for Tiny Objects (sub-type 1 - ‘Normal’) + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version, Type & Length</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Data <em>(variable size)</em></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version, Type & Length</p></td> + <td> + <p>This is a bit field with the following definition: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>6-7</code></td> + <td>The current version of ID format. This document + describes version 0. + </td> + </tr> + <tr> + <td align="center"><code>4-5</code></td> + <td>The ID type. Tiny objects have a value of <code>2</code>. + </td> + </tr> + <tr> + <td align="center"><code>0-3</code></td> + <td>The length of the tiny object. The value stored + is one less than the actual length (since zero-length + objects are not allowed to be stored in the heap). + For example, an object of actual length 1 has an + encoded length of 0, an object of actual length 2 + has an encoded length of 1, and so on. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Data</p></td> + <td> + <p>This is the data for the object. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption>Fractal Heap ID for Tiny Objects (sub-type 2 - ‘Extended’) + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version, Type & Length</td> + <td>Extended Length</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Data <em>(variable size)</em></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version, Type & Length</p></td> + <td> + <p>This is a bit field with the following definition: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>6-7</code></td> + <td>The current version of ID format. This document + describes version 0. + </td> + </tr> + <tr> + <td align="center"><code>4-5</code></td> + <td>The ID type. Tiny objects have a value of <code>2</code>. + </td> + </tr> + <tr> + <td align="center"><code>0-3</code></td> + <td>These 4 bits, together with the next byte, form an + unsigned 12-bit integer for holding the length of the + object. These 4-bits are bits 8-11 of the 12-bit integer. + See description for the <em>Extended Length</em> field below. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Extended Length</p></td> + <td> + <p>This byte, together with the 4 bits in the previous byte, + forms an unsigned 12-bit integer for holding the length of + the tiny object. These 8 bits are bits 0-7 of the 12-bit + integer formed. The value stored is one less than the actual + length (since zero-length objects are not allowed to be + stored in the heap). For example, an object of actual length + 1 has an encoded length of 0, an object of actual length + 2 has an encoded length of 1, and so on. + </p> + </td> + </tr> + + <tr> + <td><p>Data</p></td> + <td> + <p>This is the data for the object. + </p> + </td> + </tr> + + </table> + </div> + + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption>Fractal Heap ID for Huge Objects (sub-type 1 & 2): indirectly accessed, non-filtered/filtered + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version & Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />v2 B-tree Key<sup>L</sup><em> (variable size)</em><br /><br /></td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version & Type</p></td> + <td> + <p>This is a bit field with the following definition: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>6-7</code></td> + <td>The current version of ID format. This document + describes version 0. + </td> + </tr> + <tr> + <td align="center"><code>4-5</code></td> + <td>The ID type. Huge objects have a value of <code>1</code>. + </td> + </tr> + <tr> + <td align="center"><code>0-3</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>v2 B-tree Key</p></td> + <td><p>This field is the B-tree key for retrieving the information + from the version 2 B-tree for huge objects needed to access the + object. See the description of <a href="#V2Btrees">v2 B-tree</a> + records sub-type 1 & 2 for a description of the fields. New key + values are derived from <em>Next Huge Object ID</em> in the + <em>Fractal Heap Header</em>.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption>Fractal Heap ID for Huge Objects (sub-type 3): directly accessed, non-filtered + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version & Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Address <sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Length <sup>L</sup><br /><br /></td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version & Type</p></td> + <td> + <p>This is a bit field with the following definition: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>6-7</code></td> + <td>The current version of ID format. This document + describes version 0. + </td> + </tr> + <tr> + <td align="center"><code>4-5</code></td> + <td>The ID type. Huge objects have a value of <code>1</code>. + </td> + </tr> + <tr> + <td align="center"><code>0-3</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Address</p></td> + <td><p>This field is the address of the object in the file.</p> + </td> + </tr> + + <tr> + <td><p>Length</p></td> + <td><p>This field is the length of the object in the file.</p> + </td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption>Fractal Heap ID for Huge Objects (sub-type 4): directly accessed, filtered + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version & Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Address <sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Length <sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Filter Mask</td> + </tr> + + <tr> + <td colspan="4"><br />De-filtered Size <sup>L</sup><br /><br /></td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td> + </tr> + <tr> + <td> </td> + <td>(Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version & Type</p></td> + <td> + <p>This is a bit field with the following definition: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>6-7</code></td> + <td>The current version of ID format. This document + describes version 0. + </td> + </tr> + <tr> + <td align="center"><code>4-5</code></td> + <td>The ID type. Huge objects have a value of <code>1</code>. + </td> + </tr> + <tr> + <td align="center"><code>0-3</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Address</p></td> + <td><p>This field is the address of the filtered object in the file.</p> + </td> + </tr> + + <tr> + <td><p>Length</p></td> + <td><p>This field is the length of the filtered object in the file.</p> + </td> + </tr> + + <tr> + <td><p>Filter Mask</p></td> + <td><p>This field is the I/O pipeline filter mask for the + filtered object in the file.</p> + </td> + </tr> + + <tr> + <td><p>Filtered Size</p></td> + <td><p>This field is the size of the de-filtered object in the file.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption>Fractal Heap ID for Managed Objects + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version & Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4">Offset <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Length <em>(variable size)</em></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version & Type</p></td> + <td><p>This is a bit field with the following definition: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>6-7</code></td> + <td>The current version of ID format. This document + describes version 0. + </td> + </tr> + <tr> + <td align="center"><code>4-5</code></td> + <td>The ID type. Managed objects have a value of <code>0</code>. + </td> + </tr> + <tr> + <td align="center"><code>0-3</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Offset</p></td> + <td><p>This field is the offset of the object in the heap. + This field’s size is the minimum number of bytes + necessary to encode the <em>Maximum Heap Size</em> value + (from the <em>Fractal Heap Header</em>). For example, if the + value of the <em>Maximum Heap Size</em> is less than 256 bytes, + this field is 1 byte in length, a <em>Maximum Heap Size</em> + of 256-65535 bytes uses a 2 byte length, and so on.</p></td> + </tr> + + <tr> + <td><p>Length</p></td> + <td><p>This field is the length of the object in the heap. It + is determined by taking the minimum value of <em>Maximum + Direct Block Size</em> and <em>Maximum Size of Managed + Objects</em> in the <em>Fractal Heap Header</em>. Again, + the minimum number of bytes needed to encode that value is + used for the size of this field.</p></td> + </tr> + </table> + </div> + +<br /> +<h3><a name="FreeSpaceManager"> +III.G. Disk Format: Level 1G - Free-space Manager</a></h3> + + <p> + Free-space managers are used to describe space within a heap or + the entire HDF5 file that is not currently used for that heap or + file. + </p> + + <p> + The <em>free-space manager header</em> contains metadata information + about the space being tracked, along with the address of the list + of <em>free space sections</em> which actually describes the free + space. The header records information about free-space sections being + tracked, creation parameters for handling free-space sections of a + client, and section information used to locate the collection of + free-space sections. + </p> + + <p> + The <em>free-space section list</em> stores a collection of + free-space sections that is specific to each <em>client</em> of the + free-space manager. + + For example, the fractal heap is a client of the free space manager + and uses it to track unused space within the heap. There are 4 + types of section records for the fractal heap, each of which has + its own format, listed below. + </p> + + <div align="center"> + <table class="format"> + <caption> + Free-space Manager Header + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version</td> + <td>Client ID</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Total Space Tracked<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Total Number of Sections<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Number of Serialized Sections<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Number of Un-Serialized Sections<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="2">Number of Section Classes</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="2">Shrink Percent</td> + <td colspan="2">Expand Percent</td> + </tr> + + <tr> + <td colspan="2">Size of Address Space</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Maximum Section Size <sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of Serialized Section List<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Size of Serialized Section List Used<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Allocated Size of Serialized Section List<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="35%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>FSHD</code>” is used to + indicate the beginning of the Free-space Manager Header. + This gives file consistency checking utilities a better chance of + reconstructing a damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This is the version number for the Free-space Manager Header + and this document describes version 0.</p> + </td> + </tr> + + <tr> + <td><p>Client ID</p></td> + <td> + <p>This is the client ID for identifying the user of this + free-space manager: + + <table class="list"> + <tr> + <th width="20%" align="center">ID</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Fractal heap + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>File + </td> + </tr> + <tr> + <td align="center"><code>2+</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Total Space Tracked</p></td> + <td> + <p>This is the total amount of free space being tracked, in bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Total Number of Sections</p></td> + <td> + <p>This is the total number of free-space sections being tracked. + </p> + </td> + </tr> + + <tr> + <td><p>Number of Serialized Sections</p></td> + <td> + <p>This is the number of serialized free-space sections being + tracked. + </p> + </td> + </tr> + <tr> + <td><p>Number of Un-Serialized Sections</p></td> + <td> + <p>This is the number of un-serialized free-space sections being + managed. Un-serialized sections are created by the free-space + client when the list of sections is read in. + </p> + </td> + </tr> + + <tr> + <td><p>Number of Section Classes</p></td> + <td> + <p>This is the number of section classes handled by this free space + manager for the free-space client. + </p> + </td> + </tr> + + <tr> + <td><p>Shrink Percent</p></td> + <td> + <p>This is the percent of current size to shrink the allocated + serialized free-space section list. + </p> + </td> + </tr> + + <tr> + <td><p>Expand Percent</p></td> + <td> + <p>This is the percent of current size to expand the allocated + serialized free-space section list. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Address Space</p></td> + <td> + <p>This is the size of the address space that free-space sections + are within. This is stored as the log<sub>2</sub> of the + actual value (in other words, the number of bits required + to store values within that address space). + </p> + </td> + </tr> + + <tr> + <td><p>Maximum Section Size</p></td> + <td> + <p>This is the maximum size of a section to be tracked. + </p> + </td> + </tr> + + <tr> + <td><p>Address of Serialized Section List</p></td> + <td> + <p>This is the address where the serialized free-space section + list is stored. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Serialized Section List Used</p></td> + <td> + <p>This is the size of the serialized free-space section + list used (in bytes). This value must be less than + or equal to the <em>allocated size of serialized section + list</em>, below. + </p> + </td> + </tr> + + <tr> + <td><p>Allocated Size of Serialized Section List</p></td> + <td> + <p>This is the size of serialized free-space section list + actually allocated (in bytes). + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the free-space manager header.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <p>The free-space sections being managed are stored in a + <em>free-space section list</em>, described below. The sections + in the free-space section list are stored in the following way: + a count of the number of sections describing a particular size of + free space and the size of the free-space described (in bytes), + followed by a list of section description records; then another + section count and size, followed by the list of section + descriptions for that size; and so on.</p> + + + <div align="center"> + <table class="format"> + <caption> + Free-space Section List + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Free-space Manager Header Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Number of Section Records in Set #0 <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Size of Free-space Section Described in Record Set #0 <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #0 Section Record #0 Offset<em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="1">Record Set #0 Section Record #0 Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #0 Section Record #0 Data <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">Record Set #0 Section Record #K-1 Offset<em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="1">Record Set #0 Section Record #K-1 Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #0 Section Record #K-1 Data <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Number of Section Records in Set #1 <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Size of Free-space Section Described in Record Set #1 <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #1 Section Record #0 Offset<em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="1">Record Set #1 Section Record #0 Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #1 Section Record #0 Data <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">Record Set #1 Section Record #K-1 Offset<em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="1">Record Set #1 Section Record #K-1 Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #1 Section Record #K-1 Data <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4"><strong>...</strong></td> + </tr> + + <tr> + <td colspan="4"><strong>...</strong></td> + </tr> + + <tr> + <td colspan="4">Number of Section Records in Set #N-1 <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Size of Free-space Section Described in Record Set #N-1 <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #N-1 Section Record #0 Offset<em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="1">Record Set #N-1 Section Record #0 Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #N-1 Section Record #0 Data <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">Record Set #N-1 Section Record #K-1 Offset<em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="1">Record Set #N-1 Section Record #K-1 Type</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Record Set #N-1 Section Record #K-1 Data <em>(variable size)</td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="35%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>FSSE</code>” is used to + indicate the beginning of the Free-space Section Information. + This gives file consistency checking utilities a better chance of + reconstructing a damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This is the version number for the Free-space Section List + and this document describes version 0.</p> + </td> + </tr> + + <tr> + <td><p>Free-space Manager Header Address</p></td> + <td> + <p>This is the address of the <em>Free-space Manager Header</em>. + This field is principally used for file + integrity checking. + </p> + </td> + </tr> + + <tr> + <td><p>Number of Section Records for Set #N</p></td> + <td> + <p>This is the number of free-space section records for set #N. + The length of this field is the minimum number of bytes needed + to store the <em>number of serialized sections</em> (from the + <em>free-space manager header</em>). + </p> + + <p> + The number of sets of free-space section records is + determined by the <em>size of serialized section list</em> in + the <em>free-space manager header</em>. + </p> + </td> + </tr> + + <tr> + <td><p>Section Size for Record Set #N</p></td> + <td> + <p>This is the size (in bytes) of the free-space section described + for <em>all</em> the section records in set #N. + </p> + + <p> + The length of this field is the minimum number of bytes needed + to store the <em>maximum section size</em> (from the + <em>free-space manager header</em>). + </p> + </td> + </tr> + + <tr> + <td><p>Record Set #N Section #K Offset</p></td> + <td> + <p>This is the offset (in bytes) of the free-space section within + the client for the free-space manager. + </p> + + <p> + The length of this field is the minimum number of bytes needed + to store the <em>size of address space</em> (from the + <em>free-space manager header</em>). + </p> + </td> + </tr> + + <tr> + <td><p>Record Set #N Section #K Type</p></td> + <td> + <p>This is the type of the section record, used to decode the + <em>record set #N section #K data</em> information. The defined + record type for <em>file</em> client is: + + <table class="list"> + <tr> + <th width="20%" align="center">Type</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>File’s section (a range of actual bytes in file) + </td> + </tr> + <tr> + <td align="center"><code>1+</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + + <p>The defined record types for a <em>fractal heap</em> client are: + + <table class="list"> + <tr> + <th width="20%" align="center">Type</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Fractal heap “single” section + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Fractal heap “first row” section + </td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Fractal heap “normal row” section + </td> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>Fractal heap “indirect” section + </td> + </tr> + + <tr> + <td align="center"><code>4+</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Record Set #N Section #K Data</p></td> + <td> + <p>This is the section-type specific information for each record + in the record set, described below. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the <em>Free-space Section List</em>. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <p> + The section-type specific data for each free-space section record is + described below: + </p> + + <div align="center"> + <table class="format"> + <caption> + File’s Section Data Record + </caption> + + <tr> + <td colspan="4"><em>No additional record data stored</em></td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Fractal Heap “Single” Section Data Record + </caption> + + <tr> + <td colspan="4"><em>No additional record data stored</em></td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Fractal Heap “First Row” Section Data Record + </caption> + + <tr> + <td colspan="4"><em>Same format as “indirect” section data</em></td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Fractal Heap “Normal Row” Section Data Record + </caption> + + <tr> + <td colspan="4"><em>No additional record data stored</em></td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Fractal Heap “Indirect” Section Data Record + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Fractal Heap Indirect Block Offset <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="2">Block Start Row</td> + <td colspan="2">Block Start Column</td> + </tr> + + <tr> + <td colspan="2">Number of Blocks</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Fractal Heap Block Offset</p></td> + <td> + <p>The offset of the indirect block in the fractal heap’s address + space containing the empty blocks. + </p> + <p> + The number of bytes used to encode this field is the minimum + number of bytes needed to encode values for the <em>Maximum + Heap Size</em> (in the fractal heap’s header). + </p> + </td> + </tr> + + <tr> + <td><p>Block Start Row</p></td> + <td> + <p>This is the row that the empty blocks start in. + </p> + </td> + </tr> + + <tr> + <td><p>Block Start Column</p></td> + <td> + <p>This is the column that the empty blocks start in. + </p> + </td> + </tr> + + <tr> + <td><p>Number of Blocks</p></td> + <td> + <p>This is the number of empty blocks covered by the section. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h3><a name="SOHMTable"> +III.H. Disk Format: Level 1H - Shared Object Header Message Table</a></h3> + + <p> + The <em>shared object header message table</em> is used to locate + object + header messages that are shared between two or more object headers + in the file. Shared object header messages are stored and indexed + in the file in one of two ways: indexed sequentially in a + <em>shared header message list</em> or indexed with a v2 B-tree. + The shared messages themselves are either stored in a fractal + heap (when two or more objects share the message), or remain in an + object’s header (when only one object uses the message currently, + but the message can be shared in the future). + </p> + + <p> + The <em>shared object header message table</em> + contains a list of shared message index headers. Each index header + records information about the version of the index format, the index + storage type, flags for the message types indexed, the number of + messages in the index, the address where the index resides, + and the fractal heap address if shared messages are stored there. + </p> + + <p> + Each index can be either a list or a v2 B-tree and may transition + between those two forms as the number of messages in the index + varies. Each shared message record contains information used to + locate the shared message from either a fractal heap or an object + header. The types of messages that can be shared are: <em>Dataspace, + Datatype, Fill Value, Filter Pipeline and Attribute</em>. + </p> + + <p> + The <em>shared object header message table</em> is pointed to + from a <a href="#SOHMTableMessage">shared message table</a> message + in the superblock extension for a file. This message stores the + version of the table format, along with the number of index headers + in the table. + </p> + + <div align="center"> + <table class="format"> + <caption> + Shared Object Header Message Table + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td>Version for index #0</td> + <td>Index Type for index #0</td> + <td colspan="2">Message Type Flags for index #0</td> + </tr> + + <tr> + <td colspan="4">Minimum Message Size for index #0</td> + </tr> + + <tr> + <td colspan="2">List Cutoff for index #0</td> + <td colspan="2">v2 B-tree Cutoff for index #0</td> + </tr> + + <tr> + <td colspan="2">Number of Messages for index #0</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Index Address<sup>O</sup> for index #0<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Fractal Heap Address<sup>O</sup> for index #0<br /><br /></td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td>Version for index #N-1</td> + <td>Index Type for index #N-1</td> + <td colspan="2">Message Type Flags for index #N-1</td> + </tr> + + <tr> + <td colspan="4">Minimum Message Size for index #N-1</td> + </tr> + + <tr> + <td colspan="2">List Cutoff for index #N-1</td> + <td colspan="2">v2 B-tree Cutoff for index #N-1</td> + </tr> + + <tr> + <td colspan="2">Number of Messages for index #N-1</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Index Address<sup>O</sup> for index #N-1<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Fractal Heap Address<sup>O</sup> for index #N-1<br /><br /></td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="35%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>SMTB</code>” is used to + indicate the beginning of the Shared Object Header Message table. + This gives file consistency checking utilities a better chance of + reconstructing a damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version for index #N</p></td> + <td> + <p>This is the version number for the list of shared object header message + indexes and this document describes version 0.</p> + </td> + </tr> + + <tr> + <td><p>Index Type for index #N</p></td> + <td> + <p>The type of index can be an unsorted list or a v2 B-tree. + </p> + </td> + </tr> + + <tr> + <td><p>Message Type Flags for index #N</p></td> + <td> + <p>This field indicates the type of messages tracked in the index, + as follows: + <table class="list"> + <tr> + <th width="20%" align="center">Bits</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, the index tracks <em>Dataspace Messages</em>. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>If set, the message tracks <em>Datatype Messages</em>. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>If set, the message tracks <em>Fill Value Messages</em>. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>If set, the message tracks <em>Filter Pipeline Messages</em>. + </td> + </tr> + <tr> + <td align="center"><code>4</code></td> + <td>If set, the message tracks <em>Attribute Messages</em>. + </td> + </tr> + <tr> + <td align="center"><code>5-15</code></td> + <td>Reserved (zero). + </td> + </tr> + </table></p> + + + <p> + An index can track more than one type of message, but each type + of message can only by in one index. + </p> + </td> + </tr> + + <tr> + <td><p>Minimum Message Size for index #N</p></td> + <td> + <p>This is the message size sharing threshold for the index. + If the encoded size of the message is less than this value, the + message is not shared. + </p> + </td> + </tr> + + <tr> + <td><p>List Cutoff for index #N</p></td> + <td> + <p>This is the cutoff value for the indexing of messages to + switch from a list to a v2 B-tree. If the number of messages + is greater than this value, the index should be a v2 B-tree. + </p> + </td> + </tr> + <tr> + <td><p>v2 B-tree Cutoff for index #N</p></td> + <td> + <p>This is is the cutoff value for the indexing of messages to + switch from a v2 B-tree back to a list. If the number of + messages is less than this value, the index should be a list. + </p> + </td> + </tr> + + <tr> + <td><p>Number of Messages for index #N</p></td> + <td> + <p>The number of shared messages being tracked for the index. + </p> + </td> + </tr> + + <tr> + <td><p>Index Address for index #N</p></td> + <td> + <p>This field is the address of the list or v2 B-tree where the + index nodes reside. + </p> + </td> + </tr> + + <tr> + <td><p>Fractal Heap Address for index #N</p></td> + <td> + <p>This field is the address of the fractal heap if shared messages + are stored there. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the table.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <p> + Shared messages are indexed either with a <em>shared message record + list</em>, described below, or using a v2 B-tree (using record type 7). + The number of records in the <em>shared message record list</em> is + determined in the index’s entry in the <em>shared object header message + table</em>. + </p> + + <div align="center"> + <table class="format"> + <caption> + Shared Message Record List + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + + <tr> + <td colspan="4">Shared Message Record #0</td> + </tr> + + <tr> + <td colspan="4">Shared Message Record #1</td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">Shared Message Record #N-1</td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>SMLI</code>” is used to + indicate the beginning of a list of index nodes. + This gives file consistency checking utilities a better chance of + reconstructing a damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Shared Message Record #N</p></td> + <td> + <p>The record for locating the shared message, either in the + fractal heap for the index, or an object header (see format for + <em>index nodes</em> below). + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the list. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <p> + The record for each shared message in an index is stored in one of the + following forms: + </p> + + <div align="center"> + <table class="format"> + <caption> + Shared Message Record, for messages stored in a fractal heap + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Message Location</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Hash Value</td> + </tr> + + <tr> + <td colspan="4">Reference Count</td> + </tr> + + <tr> + <td colspan="4"><br />Fractal Heap ID<br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Message Location</p></td> + <td> + <p>This has a value of 0 indicating that the message is stored in + the heap. + </p> + </td> + </tr> + + <tr> + <td><p>Hash Value</p></td> + <td> + <p>This is the hash value for the message. + </p> + </td> + </tr> + + <tr> + <td><p>Reference Count</p></td> + <td> + <p>This is the number of times the message is used in the file. + </p> + </td> + </tr> + + <tr> + <td><p>Fractal Heap ID</p></td> + <td> + <p>This is an 8-byte fractal heap ID for the message as stored in + the fractal heap for the index. + </p> + </td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Shared Message Record, for messages stored in an object header + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Message Location</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Hash Value</td> + </tr> + + <tr> + <td>Reserved</td> + <td>Message Type</td> + <td colspan="2">Creation Index</td> + </tr> + + <tr> + <td colspan="4"><br />Object Header Address<sup>O</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Message Location</p></td> + <td> + <p>This has a value of 1 indicating that the message is stored in + an object header. + </p> + </td> + </tr> + + <tr> + <td><p>Hash Value</p></td> + <td> + <p>This is the hash value for the message. + </p> + </td> + </tr> + + <tr> + <td><p>Message Type</p></td> + <td> + <p>This is the message type in the object header. + </p> + </td> + </tr> + + <tr> + <td><p>Creation Index</p></td> + <td> + <p>This is the creation index of the message within the object + header. + </p> + </td> + </tr> + + <tr> + <td><p>Object Header Address</p></td> + <td> + <p>This is the address of the object header where the message is + located. + </p> + </td> + </tr> + </table> + </div> + + + +<br /> +<br /> +<hr /> +<h2><a name="DataObject"> +IV. Disk Format: Level 2 - Data Objects </a></h2> + + <p>Data objects contain the “real” user-visible information in the file. + These objects compose the scientific data and other information which + are generally thought of as “data” by the end-user. All the + other information in the file is provided as a framework for + storing and accessing these data objects. + </p> + + <p>A data object is composed of header and data + information. The header information contains the information + needed to interpret the data information for the object as + well as additional “metadata” or pointers to additional + “metadata” used to describe or annotate each object. + </p> + +<br /> +<h3><a name="ObjectHeader"> +IV.A. Disk Format: Level 2A - Data Object Headers</a></h3> + + <p>The header information of an object is designed to encompass + all of the information about an object, except for the data itself. + This information includes the dataspace, the datatype, information + about how the data is stored on disk (in external files, compressed, + broken up in blocks, and so on), as well as other information used + by the library to speed up access to the data objects or maintain + a file’s integrity. Information stored by user applications + as attributes is also stored in the object’s header. The header + of each object is not necessarily located immediately prior to the + object’s data in the file and in fact may be located in any + position in the file. The order of the messages in an object header + is not significant.</p> + + <p>Object headers are composed of a prefix and a set of messages. The + prefix contains the information needed to interpret the messages and + a small amount of metadata about the object, and the messages contain + the majority of the metadata about the object. + </p> + +<br /> +<h3><a name="ObjectHeaderPrefix"> +IV.A.1. Disk Format: Level 2A1 - Data Object Header Prefix</a></h3> + +<br /> +<h4><a name="V1ObjectHeaderPrefix"> +IV.A.1.a. Version 1 Data Object Header Prefix</a></h4> + + <p>Header messages are aligned on 8-byte boundaries for version 1 + object headers. + </p> + + <div align="center"> + <table class="format"> + <caption> + Version 1 Object Header + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Reserved (zero)</td> + <td colspan="2">Total Number of Header Messages</td> + </tr> + + <tr> + <td colspan="4">Object Reference Count</td> + </tr> + + <tr> + <td colspan="4">Object Header Size</td> + </tr> + + <tr> + <td colspan="2">Header Message Type #1</td> + <td colspan="2">Size of Header Message Data #1</td> + </tr> + + <tr> + <td>Header Message #1 Flags</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br />Header Message Data #1<br /><br /></td> + </tr> + + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + + <tr> + <td colspan="2">Header Message Type #n</td> + <td colspan="2">Size of Header Message Data #n</td> + </tr> + + <tr> + <td>Header Message #n Flags</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br />Header Message Data #n<br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This value is used to determine the format of the + information in the object header. When the format of the + object header is changed, the version number + is incremented and can be used to determine how the + information in the object header is formatted. This + is version one (1) (there was no version zero (0)) of the + object header. + </p> + </td> + </tr> + + <tr> + <td><p>Total Number of Header Messages</p></td> + <td> + <p>This value determines the total number of messages listed in + object headers for this object. This value includes the messages + in continuation messages for this object. + </p> + </td> + </tr> + + <tr> + <td><p>Object Reference Count</p></td> + <td> + <p>This value specifies the number of “hard links” to this object + within the current file. References to the object from external + files, “soft links” in this file and object references in this + file are not tracked. + </p> + </td> + </tr> + + <tr> + <td><p>Object Header Size</p></td> + <td> + <p>This value specifies the number of bytes of header message data + following this length field that contain object header messages + for this object header. This value does not include the size of + object header continuation blocks for this object elsewhere in the + file. + </p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Type</p></td> + <td> + <p>This value specifies the type of information included in the + following header message data. The message types for + header messages are defined in sections below. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Header Message #n Data</p></td> + <td> + <p>This value specifies the number of bytes of header + message data following the header message type and length + information for the current message. The size includes + padding bytes to make the message a multiple of eight + bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Flags</p></td> + <td> + <p>This is a bit field with the following definition: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, the message data is constant. This is used + for messages like the datatype message of a dataset. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>If set, the message is <em>shared</em> and stored + in another location than the object header. The Header + Message Data field contains a Shared Message + (described in the <a href="#ObjectHeaderMessages">Data Object Header Messages</a> + section below) + and the Size of Header Message Data field + contains the size of that Shared Message. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>If set, the message should not be shared. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>If set, the HDF5 decoder should fail to open this object + if it does not understand the message’s type and the file + is open with permissions allowing write access to the file. + (Normally, unknown messages can just be ignored by HDF5 + decoders) + </td> + </tr> + <tr> + <td align="center"><code>4</code></td> + <td>If set, the HDF5 decoder should set bit 5 of this + message’s flags (in other words, this bit field) + if it does not understand the message’s type + and the object is modified in any way. (Normally, + unknown messages can just be ignored by HDF5 + decoders) + </td> + </tr> + <tr> + <td align="center"><code>5</code></td> + <td>If set, this object was modified by software that did not + understand this message. + (Normally, unknown messages should just be ignored by HDF5 + decoders) (Can be used to invalidate an index or a similar + feature) + </td> + </tr> + <tr> + <td align="center"><code>6</code></td> + <td>If set, this message is shareable. + </td> + </tr> + <tr> + <td align="center"><code>7</code></td> + <td>If set, the HDF5 decoder should always fail to open this + object if it does not understand the message’s type (whether + it is open for read-only or read-write access). (Normally, + unknown messages can just be ignored by HDF5 decoders) + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Header Message #n Data</p></td> + <td> + <p>The format and length of this field is determined by the + header message type and size respectively. Some header + message types do not require any data and this information + can be eliminated by setting the length of the message to + zero. The data is padded with enough zeroes to make the + size a multiple of eight. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="V2ObjectHeaderPrefix"> +IV.A.1.b. Version 2 Data Object Header Prefix</a></h4> + + <p>Note that the “total number of messages” field has been dropped from + the data object header prefix in this version. The number of messages + in the data object header is just determined by the messages encountered + in all the object header blocks.</p> + + <p>Note also that the fields and messages in this version of data object + headers have <em>no</em> alignment or padding bytes inserted - they are + stored packed together.</p> + + <div align="center"> + <table class="format"> + <caption> + Version 2 Object Header + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + <tr> + <td>Version</td> + <td>Flags</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Access time <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4">Modification Time <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4">Change Time <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4">Birth Time <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="2">Maximum # of compact attributes <em>(optional)</em></td> + <td colspan="2">Minimum # of dense attributes <em>(optional)</em></td> + </tr> + + <tr> + <td>Size of Chunk #0 <em>(variable size)</em></td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td>Header Message Type #1</td> + <td colspan="2">Size of Header Message Data #1</td> + <td>Header Message #1 Flags</td> + </tr> + + <tr> + <td colspan="2">Header Message #1 Creation Order <em>(optional)</em></td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Header Message Data #1<br /><br /></td> + </tr> + + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + + <tr> + <td>Header Message Type #n</td> + <td colspan="2">Size of Header Message Data #n</td> + <td>Header Message #n Flags</td> + </tr> + + <tr> + <td colspan="2">Header Message #n Creation Order <em>(optional)</em></td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Header Message Data #n<br /><br /></td> + </tr> + + <tr> + <td colspan="4">Gap <em>(optional, variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>OHDR</code>” + is used to indicate the + beginning of an object header. This gives file consistency + checking utilities a better chance of reconstructing a + damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This field has a value of 2 indicating version 2 of the object header. + </p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td> + <p>This field is a bit field indicating additional information + about the object header. + <table class="list"> + <tr> + <th width="20%" align="center">Bit(s)</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0-1</code></td> + <td>This two bit field determines the size of the + <em>Size of Chunk #0</em> field. The values are: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>The <em>Size of Chunk #0</em> field is 1 byte. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>The <em>Size of Chunk #0</em> field is 2 bytes. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>The <em>Size of Chunk #0</em> field is 4 bytes. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>The <em>Size of Chunk #0</em> field is 8 bytes. + </td> + </tr> + </table></p> + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>If set, attribute creation order is tracked.</td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>If set, attribute creation order is indexed.</td> + </tr> + <tr> + <td align="center"><code>4</code></td> + <td>If set, non-default attribute storage phase change + values are stored.</td> + </tr> + <tr> + <td align="center"><code>5</code></td> + <td>If set, access, modification, change and birth times + are stored.</td> + </tr> + <tr> + <td align="center"><code>6-7</code></td> + <td>Reserved</td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Access Time</p></td> + <td> + <p>This 32-bit value represents the number of seconds after the + UNIX epoch when the object’s raw data was last accessed + (in other words, read or written). + </p> + <p>This field is present if bit 5 of <em>flags</em> is set. + </p> + </td> + </tr> + + <tr> + <td><p>Modification Time</p></td> + <td> + <p>This 32-bit value represents the number of seconds after + the UNIX epoch when the object’s raw data was last + modified (in other words, written). + </p> + <p>This field is present if bit 5 of <em>flags</em> is set. + </p> + </td> + </tr> + + <tr> + <td><p>Change Time</p></td> + <td> + <p>This 32-bit value represents the number of seconds after the + UNIX epoch when the object’s metadata was last changed. + </p> + <p>This field is present if bit 5 of <em>flags</em> is set. + </p> + </td> + </tr> + + <tr> + <td><p>Birth Time</p></td> + <td> + <p>This 32-bit value represents the number of seconds after the + UNIX epoch when the object was created. + </p> + <p>This field is present if bit 5 of <em>flags</em> is set. + </p> + </td> + </tr> + + <tr> + <td><p>Maximum # of compact attributes</p></td> + <td> + <p>This is the maximum number of attributes to store in the compact + format before switching to the indexed format. + </p> + <p>This field is present if bit 4 of <em>flags</em> is set. + </p> + </td> + </tr> + + <tr> + <td><p>Minimum # of dense attributes</p></td> + <td> + <p>This is the minimum number of attributes to store in the indexed + format before switching to the compact format. + </p> + <p>This field is present if bit 4 of <em>flags</em> is set. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Chunk #0</p></td> + <td> + <p> + This unsigned value specifies the number of bytes of header + message data following this field that contain object header + information. + </p> + <p> + This value does not include the size of object header + continuation blocks for this object elsewhere in the file. + </p> + <p> + The length of this field varies depending on bits 0 and 1 of + the <em>flags</em> field. + </p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Type</p></td> + <td> + <p>Same format as version 1 of the object header, described above. + </p> + </td> + </tr> + + <tr> + <td><p>Size of Header Message #n Data</p></td> + <td> + <p>This value specifies the number of bytes of header + message data following the header message type and length + information for the current message. The size of messages + in this version does <em>not</em> include any padding bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Flags</p></td> + <td> + <p>Same format as version 1 of the object header, described above. + </p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Creation Order</p></td> + <td> + <p>This field stores the order that a message of a given type + was created in. + </p> + <p>This field is present if bit 2 of <em>flags</em> is set. + </p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Data</p></td> + <td> + <p>Same format as version 1 of the object header, described above. + </p> + </td> + </tr> + + <tr> + <td><p>Gap</p></td> + <td> + <p>A gap in an object header chunk is inferred by the end of the + messages for the chunk before the beginning of the chunk’s + checksum. Gaps are always smaller than the size of an + object header message prefix (message type + message size + + message flags). + </p> + <p>Gaps are formed when a message (typically an attribute message) + in an earlier chunk is deleted and a message from a later + chunk that does not quite fit into the free space is moved + into the earlier chunk. + </p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the object header chunk. + </p> + </td> + </tr> + </table> + </div> + + <p>The header message types and the message data associated with + them compose the critical “metadata” about each object. Some + header messages are required for each object while others are + optional. Some optional header messages may also be repeated + several times in the header itself, the requirements and number + of times allowed in the header will be noted in each header + message description below. + </p> + + +<br /> +<h3><a name="ObjectHeaderMessages"> +IV.A.2. Disk Format: Level 2A2 - Data Object Header Messages</a></h3> + + <p>Data object header messages are small pieces of metadata that are + stored in the data object header for each object in an HDF5 file. + Data object header messages provide the metadata required to describe + an object and its contents, as well as optional pieces of metadata + that annotate the meaning or purpose of the object. + </p> + + <p>Data object header messages are either stored directly in the data + object header for the object or are shared between multiple objects + in the file. When a message is shared, a flag in the <em>Message Flags</em> + indicates that the actual <em>Message Data</em> + portion of that message is stored in another location (such as another + data object header, or a heap in the file) and the <em>Message Data</em> + field contains the information needed to locate the actual information + for the message. + </p> + + <p> + The format of shared message data is described here:</p> + + <div align="center"> + <table class="format"> + <caption> + Shared Message (Version 1) + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Type</td> + <td colspan="2">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br />Address<sup>O</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number is used when there are changes in the format + of a shared object message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Never used.</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Used by the library before version 1.6.1. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Type</p></td> + <td><p>The type of shared message location: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Message stored in another object’s header (a <em>committed</em> + message). + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Address</p></td> + <td><p>The address of the object header + containing the message to be shared.</p> + </td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Shared Message (Version 2) + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Type</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Address<sup>O</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number is used when there are changes in the format + of a shared object message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Used by the library of version 1.6.1 and after. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Type</p></td> + <td><p>The type of shared message location: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Message stored in another object’s header (a <em>committed</em> + message). + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Address</p></td> + <td><p>The address of the object header + containing the message to be shared.</p></td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Shared Message (Version 3) + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Type</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Location <em>(variable size)</em></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number indicates changes in the format of shared + object message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>Used by the library of version 1.8 and after. In this + version, the <em>Type</em> field can indicate that + the message is stored in the fractal heap. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Type</p></td> + <td><p>The type of shared message location: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Message is not shared and is not shareable. + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Message stored in file’s <em>shared object header message</em> + heap (a <em>shared</em> message). + </td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Message stored in another object’s header (a <em>committed</em> + message). + </td> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>Message stored is not shared, but is sharable. + </td> + </tr> + + </table></p> + </td> + </tr> + + <tr> + <td><p>Location</p></td> + <td><p>This field contains either a <em>Size of Offsets</em>-bytes + address of the object header + containing the message to be shared, or an 8-byte fractal heap ID + for the message in the file’s <em>shared object header message</em> + heap. + </p> + </td> + </tr> + </table> + </div> + + + <p>The following is a list of currently defined header messages: + </p> + +<br /> +<h4><a name="NILMessage">IV.A.2.a. The NIL Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> NIL</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0000</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may be repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>The NIL message is used to indicate a message which is to be + ignored when reading the header messages for a data object. + [Possibly one which has been deleted for some reason.] + </td></tr> + <tr><td colspan="2"><b>Format of Data:</b> Unspecified</td></tr> + </table></center> + <!-- end msgdesc table --> + + +<br /> +<h4><a name="DataspaceMessage">IV.A.2.b. The Dataspace Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Dataspace</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0001</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies according to the number of + dimensions, as described in the following table.</td></tr> + <tr><td colspan="2"><b>Status:</b> Required for dataset objects; + may not be repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>The dataspace message describes the number of dimensions (in + other words, “rank”) and size of each dimension that + the data object has. This message is only used for datasets which + have a simple, rectilinear, array-like layout; datasets requiring + a more complex layout are not yet supported. + </td> + </tr> + + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Dataspace Message - Version 1 + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Dimensionality</td> + <td>Flags</td> + <td>Reserved</td> + </tr> + + <tr> + <td colspan="4">Reserved</td> + </tr> + + <tr> + <td colspan="4"><br />Dimension #1 Size<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4"><br />Dimension #n Size<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Dimension #1 Maximum Size<sup>L</sup> <em>(optional)</em><br /><br /></td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4"><br />Dimension #n Maximum Size<sup>L</sup> <em>(optional)</em><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Permutation Index #1<sup>L</sup> <em>(optional)</em><br /><br /></td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4"><br />Permutation Index #n<sup>L</sup> <em>(optional)</em><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This value is used to determine the format of the + Dataspace Message. When the format of the + information in the message is changed, the version number + is incremented and can be used to determine how the + information in the object header is formatted. This + document describes version one (1) (there was no version + zero (0)). + </p> + </td> + </tr> + + <tr> + <td><p>Dimensionality</p></td> + <td> + <p>This value is the number of dimensions that the data + object has. + </p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td> + <p>This field is used to store flags to indicate the + presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum + dimensions are present. Bit 1 is used to indicate that + permutation indices are present. + </p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Size</p></td> + <td> + <p>This value is the current size of the dimension of the + data as stored in the file. The first dimension stored in + the list of dimensions is the slowest changing dimension + and the last dimension stored is the fastest changing + dimension. + </p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Maximum Size</p></td> + <td> + <p>This value is the maximum size of the dimension of the + data as stored in the file. This value may be the special + “<a href="#UnlimitedDim">unlimited</a>” size which indicates + that the data may expand along this dimension indefinitely. + If these values are not stored, the maximum size of each + dimension is assumed to be the dimension’s current size. + </p> + </td> + </tr> + + <tr> + <td><p>Permutation Index #n</p></td> + <td> + <p>This value is the index permutation used to map + each dimension from the canonical representation to an + alternate axis for each dimension. If these values are + not stored, the first dimension stored in the list of + dimensions is the slowest changing dimension and the last + dimension stored is the fastest changing dimension. + </p> + </td> + </tr> + </table> + </div> + + + + <br /> + <p>Version 2 of the dataspace message dropped the optional + permutation index value support, as it was never implemented in the + HDF5 Library:</p> + + <div align="center"> + <table class="format"> + <caption> + Dataspace Message - Version 2 + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Dimensionality</td> + <td>Flags</td> + <td>Type</td> + </tr> + + <tr> + <td colspan="4"><br />Dimension #1 Size<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4"><br />Dimension #n Size<sup>L</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Dimension #1 Maximum Size<sup>L</sup> <em>(optional)</em><br /><br /></td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4"><br />Dimension #n Maximum Size<sup>L</sup> <em>(optional)</em><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>This value is used to determine the format of the + Dataspace Message. This field should be ‘2’ for version 2 + format messages. + </p> + </td> + </tr> + + <tr> + <td><p>Dimensionality</p></td> + <td> + <p>This value is the number of dimensions that the data object has. + </p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td> + <p>This field is used to store flags to indicate the + presence of parts of this message. Bit 0 (the least + significant bit) is used to indicate that maximum + dimensions are present. + </p> + </td> + </tr> + + <tr> + <td><p>Type</p></td> + <td> + <p>This field indicates the type of the dataspace: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>A <em>scalar</em> dataspace; in other words, + a dataspace with a single, dimensionless element. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>A <em>simple</em> dataspace; in other words, + a dataspace with a rank > 0 and an appropriate # of + dimensions. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>A <em>null</em> dataspace; in other words, + a dataspace with no elements. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Size</p></td> + <td> + <p>This value is the current size of the dimension of the + data as stored in the file. The first dimension stored in + the list of dimensions is the slowest changing dimension + and the last dimension stored is the fastest changing + dimension. + </p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Maximum Size</p></td> + <td> + <p>This value is the maximum size of the dimension of the + data as stored in the file. This value may be the special + “<a href="#UnlimitedDim">unlimited</a>” size which indicates + that the data may expand along this dimension indefinitely. + If these values are not stored, the maximum size of each + dimension is assumed to be the dimension’s current size. + </p> + </td> + </tr> + + </table> + </div> + + + +<!-- +<br /> +<h4><a name="DataSpaceMessage">Header Message Name: Complex Dataspace (Fiber Bundle?)</a></h4> + + <!-- start msgdesc table -- + <center> + <table class="msgdesc"> + <p><b>Header Message Name: ???????</b></td></tr> + <b>Header Message Type: </b>0x0002<br /> + <b>Length:</b> Varies</td></tr> + + <b>Status:</b> One of the <em>Simple Dataspace</em> or + <em>Complex Dataspace</em> messages is required (but not both) and may + not be repeated.<br /> <b>Description:</b> The + <em>Dataspace</em> message describes space that the dataset is + mapped onto in a more comprehensive way than the <em>Simple + Dimensionality</em> message is capable of handling. The + dataspace of a dataset encompasses the type of coordinate system + used to locate the dataset’s elements as well as the structure and + regularity of the coordinate system. The dataspace also + describes the number of dimensions which the dataset inhabits as + well as a possible higher dimensional space in which the dataset + is located within. + + <br /> + <p><b>Format of Data:</b></p> + + <center> + <table border cellpadding="4" width="80%"> + <caption align="bottom"> + <b>HDF5 Dataspace Message Layout</b> + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="4">Mesh Type</td> + </tr> + <tr align="center"> + <td colspan="4">Logical Dimensionality</td> + </tr> + </table> + </center> + + <br /> + <dl> + <dt>The elements of the dimensionality message are described below: + <dd> + <dl> + <dt>Mesh Type: (unsigned 32-bit integer) + <dd>This value indicates whether the grid is + polar/spherical/cartesion, + structured/unstructured and regular/irregular. <br /> + The mesh type value is broken up as follows: <br /> + + <br /> + <center> + <table border cellpadding="4" width="80%"> + <caption align="bottom"> + <b>HDF5 Mesh-type Layout</b> + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1">Mesh Embedding</td> + <td colspan="1">Coordinate System</td> + <td colspan="1">Structure</td> + <td colspan="1">Regularity</td> + </tr> + </table> + </center> + The following are the definitions of mesh-type bytes: + <dl> + <dt>Mesh Embedding + <dd>This value indicates whether the dataset dataspace + is located within + another dataspace or not: + <dl> <dl> + <dt><STANDALONE> + <dd>The dataset mesh is self-contained and is not + embedded in another mesh. + <dt><EMBEDDED> + <dd>The dataset’s dataspace is located within + another dataspace, as + described in information below. + </dl> </dl> + <dt>Coordinate System + <dd>This value defines the type of coordinate system + used for the mesh: + <dl> <dl> + <dt><POLAR> + <dd>The last two dimensions are in polar + coordinates, higher dimensions are + cartesian. + <dt><SPHERICAL> + <dd>The last three dimensions are in spherical + coordinates, higher dimensions + are cartesian. + <dt><CARTESIAN> + <dd>All dimensions are in cartesian coordinates. + </dl> </dl> + <dt>Structure + <dd>This value defines the locations of the grid-points + on the axes: + <dl> <dl> + <dt><STRUCTURED> + <dd>All grid-points are on integral, sequential + locations, starting from 0. + <dt><UNSTRUCTURED> + <dd>Grid-points locations in each dimension are + explicitly defined and + may be of any numeric datatype. + </dl> </dl> + <dt>Regularity + <dd>This value defines the locations of the dataset + points on the grid: + <dl> <dl> + <dt><REGULAR> + <dd>All dataset elements are located at the + grid-points defined. + <dt><IRREGULAR> + <dd>Each dataset element has a particular + grid-location defined. + </dl> </dl> + </dl> + <p>The following grid combinations are currently allowed:</p> + <dl> <dl> + <dt><POLAR-STRUCTURED-REGULAR> + <dt><SPHERICAL-STRUCTURED-REGULAR> + <dt><CARTESIAN-STRUCTURED-REGULAR> + <dt><POLAR-UNSTRUCTURED-REGULAR> + <dt><SPHERICAL-UNSTRUCTURED-REGULAR> + <dt><CARTESIAN-UNSTRUCTURED-REGULAR> + <dt><CARTESIAN-UNSTRUCTURED-IRREGULAR> + </dl> </dl> + All of the above grid types can be embedded within another + dataspace. + <br /> <br /> + <dt>Logical Dimensionality: (unsigned 32-bit integer) + <dd>This value is the number of dimensions that the dataset occupies. + + <br /> + <center> + <table border cellpadding="4" width="80%"> + <caption align="bottom"> + <b>HDF5 Dataspace Embedded Dimensionality Information</b> + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="4">Embedded Dimensionality</td> + </tr> + <tr align="center"> + <td colspan="4">Embedded Dimension Size #1</td> + </tr> + <tr align="center"> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr align="center"> + <td colspan="4">Embedded Dimension Size #n</td> + </tr> + <tr align="center"> + <td colspan="4">Embedded Origin Location #1</td> + </tr> + <tr align="center"> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr align="center"> + <td colspan="4">Embedded Origin Location #n</td> + </tr> + </table> + </center> + + <dt>Embedded Dimensionality: (unsigned 32-bit integer) + <dd>This value is the number of dimensions of the space the + dataset is located within: in other words, a planar dataset + located within a 3-D space, a 3-D dataset + which is a subset of another 3-D space, and so on. + <dt>Embedded Dimension Size: (unsigned 32-bit integer) + <dd>These values are the sizes of the dimensions of the + embedded dataspace + that the dataset is located within. + <dt>Embedded Origin Location: (unsigned 32-bit integer) + <dd>These values comprise the location of the dataset’s + origin within the embedded dataspace. + </dl> + </dl> + [Comment: need some way to handle different orientations of the + dataset dataspace + within the embedded dataspace]<br /> + + <br /> + <center> + <table border cellpadding="4" width="80%"> + <caption align="bottom"> + <b>HDF5 Dataspace Structured/Regular Grid Information</b> + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="4">Logical Dimension Size #1</td> + </tr> + <tr align="center"> + <td colspan="4">Logical Dimension Maximum #1</td> + </tr> + <tr align="center"> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr align="center"> + <td colspan="4">Logical Dimension Size #n</td> + </tr> + <tr align="center"> + <td colspan="4">Logical Dimension Maximum #n</td> + </tr> + </table> + </center> + + <br /> + <dl> + <dt>The elements of the dimensionality message are described below: + <dd> + <dl> + <dt>Logical Dimension Size #n: (unsigned 32-bit integer) + <dd>This value is the current size of the dimension of the + data as stored in + the file. The first dimension stored in the list of + dimensions is the slowest + changing dimension and the last dimension stored is the + fastest changing + dimension. + <dt>Logical Dimension Maximum #n: (unsigned 32-bit integer) + <dd>This value is the maximum size of the dimension of the + data as stored in + the file. This value may be the special value + <UNLIMITED> which + indicates that the data may expand along this dimension + indefinitely. + </dl> + </dl> + <br /> + <center> + <table border cellpadding="4" width="80%"> + <caption align="bottom"> + <b>HDF5 Dataspace Structured/Irregular Grid Information</b> + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="4"># of Grid Points in Dimension #1</td> + </tr> + <tr align="center"> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr align="center"> + <td colspan="4"># of Grid Points in Dimension #n</td> + </tr> + <tr align="center"> + <td colspan="4">Datatype of Grid Point Locations</td> + </tr> + <tr align="center"> + <td colspan="4">Location of Grid Points in Dimension #1</td> + </tr> + <tr align="center"> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr align="center"> + <td colspan="4">Location of Grid Points in Dimension #n</td> + </tr> + </table> + </center> + + <br /> + <center> + <table border cellpadding="4" width="80%"> + <caption align="bottom"> + <b>HDF5 Dataspace Unstructured Grid Information</b> + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="4"># of Grid Points</td> + </tr> + <tr align="center"> + <td colspan="4">Datatype of Grid Point Locations</td> + </tr> + <tr align="center"> + <td colspan="4">Grid Point Locations<br />.<br />.<br /></td> + </tr> + </table> + </center> +--> + +<br /> +<h4><a name="LinkInfoMessage">IV.A.2.c. The Link Info Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Link Info</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x002 </td></tr> + <tr><td colspan="2"><b>Length:</b> Varies </td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated. </td></tr> + <tr><td><b>Description:</b></td> + <td>The link info message tracks variable information about the + current state of the links for a “new style” + group’s behavior. Variable information will be stored in + this message and constant information will be stored in the + <a href="#GroupInfoMessage">Group Info</a> message. + </td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Link Info + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Flags</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Maximum Creation Index <em>(8 bytes, optional)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Fractal Heap Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of v2 B-tree for Name Index<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Address of v2 B-tree for Creation Order Index<sup>O</sup> <em>(optional)</em><br /><br /></td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number for this message. This document describes + version 0.</p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>This field determines various optional aspects of the link + info message: + + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, creation order for the links is tracked. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>If set, creation order for the links is indexed. + </td> + </tr> + <tr> + <td align="center"><code>2-7</code></td> + <td>Reserved</td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Maximum Creation Index</p></td> + <td><p>This 64-bit value is the maximum creation order index value + stored for a link in this group.</p> + <p>This field is present if bit 0 of <em>flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Fractal Heap Address</p></td> + <td> + <p> + This is the address of the fractal heap to store dense links. + Each link stored in the fractal heap is stored as a + <a href="#LinkMessage">Link Message</a>. + </p> + <p> + If there are no links in the group, or the group’s links + are stored “compactly” (as object header messages), this + value will be the <a href="#UndefinedAddress">undefined + address</a>. + </p> + </td> + </tr> + + <tr> + <td><p>Address of v2 B-tree for Name Index</p></td> + <td><p>This is the address of the version 2 B-tree to index names of links.</p> + <p>If there are no links in the group, or the group’s links + are stored “compactly” (as object header messages), this + value will be the <a href="#UndefinedAddress">undefined + address</a>. + </p> + </td> + </tr> + + <tr> + <td><p>Address of v2 B-tree for Creation Order Index</p></td> + <td><p>This is the address of the version 2 B-tree to index creation order of links.</p> + <p>If there are no links in the group, or the group’s links + are stored “compactly” (as object header messages), this + value will be the <a href="#UndefinedAddress">undefined + address</a>. + </p> + <p>This field exists if bit 1 of <em>flags</em> is set.</p> + </td> + </tr> + + </table> + </div> + + +<br /> +<h4><a name="DatatypeMessage">IV.A.2.d. The Datatype Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Datatype</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0003 + </td></tr> + <tr><td colspan="2"><b>Length:</b> Variable</td></tr> + <tr><td colspan="2"><b>Status:</b> Required for dataset or committed + datatype (formerly named datatype) objects; may not be repeated. + </td></tr> + <tr><td><b>Description:</b></td> + <td><p>The datatype message defines the datatype for each element + of a dataset or a common datatype for sharing between multiple + datasets. A datatype can describe an atomic type like a fixed- + or floating-point type or more complex types like a C struct + (compound datatype), array (array datatype) or C++ vector + (variable-length datatype).</p> + <p>Datatype messages that are part of a dataset object do not + describe how elements are related to one another; the dataspace + message is used for that purpose. Datatype messages that are part of + a committed datatype (formerly named datatype) message describe + a common datatype that can be shared by multiple datasets in the + file.</p> + </td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Datatype Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Class and Version</td> + <td>Class Bit Field, Bits 0-7</td> + <td>Class Bit Field, Bits 8-15</td> + <td>Class Bit Field, Bits 16-23</td> + </tr> + + <tr> + <td colspan="4">Size</td> + </tr> + + <tr> + <td colspan="4"><br /><br />Properties<br /><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Class and Version</p></td> + <td> + <p>The version of the datatype message and the datatype’s class + information are packed together in this field. The version + number is packed in the top 4 bits of the field and the class + is contained in the bottom 4 bits. + </p> + <p>The version number information is used for changes in the + format of the datatype message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Never used + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>Used by early versions of the library to encode + compound datatypes with explicit array fields. + See the compound datatype description below for + further details. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>Used when an array datatype needs to be encoded. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>Used when a VAX byte-ordered type needs to be + encoded. Packs various other datatype classes more + efficiently also. + </td> + </tr> + </table></p> + + <p>The class of the datatype determines the format for the class + bit field and properties portion of the datatype message, which + are described below. The + following classes are currently defined: + + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Fixed-Point</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Floating-Point</td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Time</td> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>String</td> + </tr> + + <tr> + <td align="center"><code>4</code></td> + <td>Bit field</td> + </tr> + + <tr> + <td align="center"><code>5</code></td> + <td>Opaque</td> + </tr> + + <tr> + <td align="center"><code>6</code></td> + <td>Compound</td> + </tr> + + <tr> + <td align="center"><code>7</code></td> + <td>Reference</td> + </tr> + + <tr> + <td align="center"><code>8</code></td> + <td>Enumerated</td> + </tr> + + <tr> + <td align="center"><code>9</code></td> + <td>Variable-Length</td> + </tr> + + <tr> + <td align="center"><code>10</code></td> + <td>Array</td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Class Bit Fields</p></td> + <td> + <p>The information in these bit fields is specific to each datatype + class and is described below. All bits not defined for a + datatype class are set to zero. + </p> + </td> + </tr> + + <tr> + <td><p>Size</p></td> + <td> + <p>The size of a datatype element in bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Properties</p></td> + <td> + <p>This variable-sized sequence of bytes encodes information + specific to each datatype class and is described for each class + below. If there is no property information specified for a + datatype class, the size of this field is zero bytes. + </p> + </td> + </tr> + + </table> + </div> + + + <br /> + <p>Class specific information for Fixed-Point Numbers (Class 0):</p> + + <div align="center"> + <table class="desc"> + <caption> + Fixed-point Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0</p></td> + <td><p><b>Byte Order.</b> If zero, byte order is little-endian; + otherwise, byte order is big endian.</p></td> + </tr> + + <tr> + <td><p>1, 2</p></td> + <td><p><b>Padding type.</b> Bit 1 is the lo_pad bit and bit 2 + is the hi_pad bit. If a datum has unused bits at either + end, then the lo_pad or hi_pad bit is copied to those + locations.</p></td> + </tr> + + <tr> + <td><p>3</p></td> + <td><p><b>Signed.</b> If this bit is set then the fixed-point + number is in 2’s complement form.</p></td> + </tr> + + <tr> + <td><p>4-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Fixed-Point Property Description + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="2">Bit Offset</td> + <td colspan="2">Bit Precision</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Bit Offset</p></td> + <td> + <p>The bit offset of the first significant bit of the fixed-point + value within the datatype. The bit offset specifies the number + of bits “to the right of” the value (which are set to the + lo_pad bit value). + </p> + </td> + </tr> + + <tr> + <td><p>Bit Precision</p></td> + <td> + <p>The number of bits of precision of the fixed-point value + within the datatype. This value, combined with the datatype + element’s size and the Bit Offset field specifies the number + of bits “to the left of” the value (which are set to the + hi_pad bit value). + </p> + </td> + </tr> + + </table> + </div> + + + <br /> + <p>Class specific information for Floating-Point Numbers (Class 1):</p> + + <div align="center"> + <table class="desc"> + <caption> + Floating-Point Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0, 6</p></td> + <td><p><b>Byte Order.</b> These two non-contiguous bits specify the + “endianness” of the bytes in the datatype element. + <table class="list"> + <tr> + <th width="10%" align="center">Bit 6</th> + <th width="10%" align="center">Bit 0</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td align="center"><code>0</code></td> + <td>Byte order is little-endian + </td> + </tr> + <tr> + <td align="center"><code>0</code></td> + <td align="center"><code>1</code></td> + <td>Byte order is big-endian + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td align="center"><code>0</code></td> + <td>Reserved + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td align="center"><code>1</code></td> + <td>Byte order is VAX-endian + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>1, 2, 3</p></td> + <td><p><b>Padding type.</b> Bit 1 is the low bits pad type, bit 2 + is the high bits pad type, and bit 3 is the internal bits + pad type. If a datum has unused bits at either end or between + the sign bit, exponent, or mantissa, then the value of bit + 1, 2, or 3 is copied to those locations.</p></td> + </tr> + + <tr> + <td><p>4-5</p></td> + <td><p><b>Mantissa Normalization.</b> This 2-bit bit field specifies + how the most significant bit of the mantissa is managed. + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>No normalization + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>The most significant bit of the mantissa is always set + (except for 0.0). + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>The most significant bit of the mantissa is not stored, + but is implied to be set. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>Reserved. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>7</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + + <tr> + <td><p>8-15</p></td> + <td><p><b>Sign Location.</b> This is the bit position of the sign + bit. Bits are numbered with the least significant bit zero.</p></td> + </tr> + + <tr> + <td><p>16-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Floating-Point Property Description + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="2">Bit Offset</td> + <td colspan="2">Bit Precision</td> + </tr> + + <tr> + <td>Exponent Location</td> + <td>Exponent Size</td> + <td>Mantissa Location</td> + <td>Mantissa Size</td> + </tr> + + <tr> + <td colspan="4">Exponent Bias</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Bit Offset</p></td> + <td> + <p>The bit offset of the first significant bit of the floating-point + value within the datatype. The bit offset specifies the number + of bits “to the right of” the value. + </p> + </td> + </tr> + + <tr> + <td><p>Bit Precision</p></td> + <td> + <p>The number of bits of precision of the floating-point value + within the datatype. + </p> + </td> + </tr> + + <tr> + <td><p>Exponent Location</p></td> + <td> + <p>The bit position of the exponent field. Bits are numbered with + the least significant bit number zero. + </p> + </td> + </tr> + + <tr> + <td><p>Exponent Size</p></td> + <td> + <p>The size of the exponent field in bits. + </p> + </td> + </tr> + + <tr> + <td><p>Mantissa Location</p></td> + <td> + <p>The bit position of the mantissa field. Bits are numbered with + the least significant bit number zero. + </p> + </td> + </tr> + + <tr> + <td><p>Mantissa Size</p></td> + <td> + <p>The size of the mantissa field in bits. + </p> + </td> + </tr> + + <tr> + <td><p>Exponent Bias</p></td> + <td> + <p>The bias of the exponent field. + </p> + </td> + </tr> + + </table> + </div> + + + <br /> + <p>Class specific information for Time (Class 2):</p> + + + <div align="center"> + <table class="desc"> + <caption> + Time Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0</p></td> + <td><p><b>Byte Order.</b> If zero, byte order is little-endian; + otherwise, byte order is big endian.</p></td> + </tr> + + <tr> + <td><p>1-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Time Property Description + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="2">Bit Precision</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Bit Precision</p></td> + <td> + <p>The number of bits of precision of the time value. + </p> + </td> + </tr> + + </table> + </div> + + + <br /> + <p>Class specific information for Strings (Class 3):</p> + + + <div align="center"> + <table class="desc"> + <caption> + String Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0-3</p></td> + <td><p><b>Padding type.</b> This four-bit value determines the + type of padding to use for the string. The values are: + + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Null Terminate: A zero byte marks the end of the + string and is guaranteed to be present after + converting a long string to a short string. When + converting a short string to a long string the value is + padded with additional null characters as necessary. + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Null Pad: Null characters are added to the end of + the value during conversions from short values to long + values but conversion in the opposite direction simply + truncates the value. + </td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Space Pad: Space characters are added to the end of + the value during conversions from short values to long + values but conversion in the opposite direction simply + truncates the value. This is the Fortran + representation of the string. + </td> + </tr> + + <tr> + <td align="center"><code>3-15</code></td> + <td>Reserved + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>4-7</p></td> + <td><p><b>Character Set.</b> The character set used to + encode the string. + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>ASCII character set encoding + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>UTF-8 character set encoding + </td> + </tr> + + <tr> + <td align="center"><code>2-15</code></td> + <td>Reserved + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>8-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <p>There are no properties defined for the string class. + </p> + + + <p>Class specific information for bit fields (Class 4):</p> + + <div align="center"> + <table class="desc"> + <caption> + Bitfield Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0</p></td> + <td><p><b>Byte Order.</b> If zero, byte order is little-endian; + otherwise, byte order is big endian.</p></td> + </tr> + + <tr> + <td><p>1, 2</p></td> + <td><p><b>Padding type.</b> Bit 1 is the lo_pad type and bit 2 + is the hi_pad type. If a datum has unused bits at either + end, then the lo_pad or hi_pad bit is copied to those + locations.</p></td> + </tr> + + <tr> + <td><p>3-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Bit Field Property Description + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="2">Bit Offset</td> + <td colspan="2">Bit Precision</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Bit Offset</p></td> + <td> + <p>The bit offset of the first significant bit of the bit field + within the datatype. The bit offset specifies the number + of bits “to the right of” the value. + </p> + </td> + </tr> + + <tr> + <td><p>Bit Precision</p></td> + <td> + <p>The number of bits of precision of the bit field + within the datatype. + </p> + </td> + </tr> + </table> + </div> + + + <br /> + <p>Class specific information for Opaque (Class 5):</p> + + <div align="center"> + <table class="desc"> + <caption> + Opaque Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0-7</p></td> + <td><p>Length of ASCII tag in bytes.</p></td> + </tr> + + <tr> + <td><p>8-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Opaque Property Description + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="4"><br />ASCII Tag<br /> + <br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>ASCII Tag</p></td> + <td> + <p>This NUL-terminated string provides a description for the + opaque type. It is NUL-padded to a multiple of 8 bytes. + </p> + </td> + </tr> + </table> + </div> + + + <br /> + <p>Class specific information for Compound (Class 6):</p> + + <div align="center"> + <table class="desc"> + <caption> + Compound Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0-15</p></td> + <td><p><b>Number of Members.</b> This field contains the number + of members defined for the compound datatype. The member + definitions are listed in the Properties field of the data + type message.</p></td> + </tr> + + <tr> + <td><p>16-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + + <p>The Properties field of a compound datatype is a list of the + member definitions of the compound datatype. The member + definitions appear one after another with no intervening bytes. + The member types are described with a (recursively) encoded datatype + message.</p> + + <p>Note that the property descriptions are different for different + versions of the datatype version. Additionally note that the version + 0 datatype encoding is deprecated and has been replaced with later + encodings in versions of the HDF5 Library from the 1.4 release + onward.</p> + + + <div align="center"> + <table class="format"> + <caption> + Compound Properties Description for Datatype Version 1 + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="4"><br />Name<br /><br /></td> + </tr> + + <tr> + <td colspan="4">Byte Offset of Member</td> + </tr> + + <tr> + <td>Dimensionality</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4">Dimension Permutation</td> + </tr> + + <tr> + <td colspan="4">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4">Dimension #1 Size (required)</td> + </tr> + + <tr> + <td colspan="4">Dimension #2 Size (required)</td> + </tr> + + <tr> + <td colspan="4">Dimension #3 Size (required)</td> + </tr> + + <tr> + <td colspan="4">Dimension #4 Size (required)</td> + </tr> + + <tr> + <td colspan="4"><br />Member Type Message<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Name</p></td> + <td> + <p>This NUL-terminated string provides a description for the + opaque type. It is NUL-padded to a multiple of 8 bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Byte Offset of Member</p></td> + <td> + <p>This is the byte offset of the member within the datatype. + </p> + </td> + </tr> + + <tr> + <td><p>Dimensionality</p></td> + <td> + <p>If set to zero, this field indicates a scalar member. If set + to a value greater than zero, this field indicates that the + member is an array of values. For array members, the size of + the array is indicated by the ‘Size of Dimension n’ field in + this message. + </p> + </td> + </tr> + + <tr> + <td><p>Dimension Permutation</p></td> + <td> + <p>This field was intended to allow an array field to have + its dimensions permuted, but this was never implemented. + This field should always be set to zero. + </p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Size</p></td> + <td> + <p>This field is the size of a dimension of the array field as + stored in the file. The first dimension stored in the list of + dimensions is the slowest changing dimension and the last + dimension stored is the fastest changing dimension. + </p> + </td> + </tr> + + <tr> + <td><p>Member Type Message</p></td> + <td> + <p>This field is a datatype message describing the datatype of + the member. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Compound Properties Description for Datatype Version 2 + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="4"><br />Name<br /><br /></td> + </tr> + + <tr> + <td colspan="4">Byte Offset of Member</td> + </tr> + + <tr> + <td colspan="4"><br />Member Type Message<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Name</p></td> + <td> + <p>This NUL-terminated string provides a description for the + opaque type. It is NUL-padded to a multiple of 8 bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Byte Offset of Member</p></td> + <td> + <p>This is the byte offset of the member within the datatype. + </p> + </td> + </tr> + + <tr> + <td><p>Member Type Message</p></td> + <td> + <p>This field is a datatype message describing the datatype of + the member. + </p> + </td> + </tr> + + </table> + </div> + + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Compound Properties Description for Datatype Version 3 + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="4"><br />Name<br /><br /></td> + </tr> + + <tr> + <td colspan="4">Byte Offset of Member <em>(variable size)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Member Type Message<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Name</p></td> + <td><p>This NUL-terminated string provides a description for the + opaque type. It is <em>not</em> NUL-padded to a multiple of 8 + bytes.</p></td> + </tr> + + <tr> + <td><p>Byte Offset of Member</p></td> + <td><p>This is the byte offset of the member within the datatype. + The field size is the minimum number of bytes necessary, + based on the size of the datatype element. For example, a + datatype element size of less than 256 bytes uses a 1 byte + length, a datatype element size of 256-65535 bytes uses a + 2 byte length, and so on.</p></td> + </tr> + + <tr> + <td><p>Member Type Message</p></td> + <td><p>This field is a datatype message describing the datatype of + the member.</p></td> + </tr> + + </table> + </div> + + + <br /> + <p>Class specific information for Reference (Class 7):</p> + + <div align="center"> + <table class="desc"> + <caption> + Reference Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0-3</p></td> + <td><p><b>Type.</b> This four-bit value contains the type of reference + described. The values defined are: + + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Object Reference: A reference to another object in this + HDF5 file. + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Dataset Region Reference: A reference to a region within + a dataset in this HDF5 file. + </td> + </tr> + + <tr> + <td align="center"><code>2-15</code></td> + <td>Reserved + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>4-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <p>There are no properties defined for the reference class. + </p> + + + <br /> + <p>Class specific information for Enumeration (Class 8):</p> + + <div align="center"> + <table class="desc"> + <caption> + Enumeration Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0-15</p></td> + <td><p><b>Number of Members.</b> The number of name/value + pairs defined for the enumeration type.</p></td> + </tr> + + <tr> + <td><p>16-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Enumeration Property Description for Datatype Versions 1 & 2 + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="4"><br />Base Type<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Names<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Values<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Base Type</p></td> + <td> + <p>Each enumeration type is based on some parent type, usually an + integer. The information for that parent type is described + recursively by this field. + </p> + </td> + </tr> + + <tr> + <td><p>Names</p></td> + <td> + <p>The name for each name/value pair. Each name is stored as a null + terminated ASCII string in a multiple of eight bytes. The names + are in no particular order. + </p> + </td> + </tr> + + <tr> + <td><p>Values</p></td> + <td> + <p>The list of values in the same order as the names. The values + are packed (no inter-value padding) and the size of each value + is determined by the parent type. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Enumeration Property Description for Datatype Version 3 + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="4"><br />Base Type<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Names<br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Values<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Base Type</p></td> + <td> + <p>Each enumeration type is based on some parent type, usually an + integer. The information for that parent type is described + recursively by this field. + </p> + </td> + </tr> + + <tr> + <td><p>Names</p></td> + <td> + <p>The name for each name/value pair. Each name is stored as a null + terminated ASCII string, <em>not</em> padded to a multiple of + eight bytes. The names are in no particular order. + </p> + </td> + </tr> + + <tr> + <td><p>Values</p></td> + <td> + <p>The list of values in the same order as the names. The values + are packed (no inter-value padding) and the size of each value + is determined by the parent type. + </p> + </td> + </tr> + + </table> + </div> + + + + <br /> + <p>Class specific information for Variable-Length (Class 9):</p> + + <div align="center"> + <table class="desc"> + <caption> + Variable-Length Bit Field Description + </caption> + + <tr> + <th width="10%">Bits</th> + <th>Meaning</th> + </tr> + + <tr> + <td><p>0-3</p></td> + <td><p><b>Type.</b> This four-bit value contains the type of + variable-length datatype described. The values defined are: + + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Sequence: A variable-length sequence of any datatype. + Variable-length sequences do not have padding or + character set information. + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>String: A variable-length sequence of characters. + Variable-length strings have padding and character set + information. + </td> + </tr> + + <tr> + <td align="center"><code>2-15</code></td> + <td>Reserved + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>4-7</p></td> + <td><p><b>Padding type.</b> (variable-length string only) + This four-bit value determines the type of padding + used for variable-length strings. The values are the same + as for the string padding type, as follows: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Null terminate: A zero byte marks the end of a string + and is guaranteed to be present after converting a long + string to a short string. When converting a short string + to a long string, the value is padded with additional null + characters as necessary. + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Null pad: Null characters are added to the end of the + value during conversion from a short string to a longer + string. Conversion from a long string to a shorter string + simply truncates the value. + </td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Space pad: Space characters are added to the end of the + value during conversion from a short string to a longer + string. Conversion from a long string to a shorter string + simply truncates the value. This is the Fortran + representation of the string. + </td> + </tr> + + <tr> + <td align="center"><code>3-15</code></td> + <td>Reserved + </td> + </tr> + </table></p> + + <p>This value is set to zero for variable-length sequences.</p> + + </td> + </tr> + + <tr> + <td><p>8-11</p></td> + <td><p><b>Character Set.</b> (variable-length string only) + This four-bit value specifies the character set + to be used for encoding the string: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>ASCII character set encoding + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>UTF-8 character set encoding + </td> + </tr> + + <tr> + <td align="center"><code>2-15</code></td> + <td>Reserved + </td> + </tr> + </table></p> + + <p>This value is set to zero for variable-length sequences.</p> + + </td> + </tr> + + <tr> + <td><p>12-23</p></td> + <td><p>Reserved (zero).</p></td> + </tr> + </table> + </div> + + <br /> + <br /> + <div align="center"> + <table class="format"> + <caption> + Variable-Length Property Description + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td colspan="4"><br />Base Type<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="10%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Base Type</p></td> + <td> + <p>Each variable-length type is based on some parent type. The + information for that parent type is described recursively by + this field. + </p> + </td> + </tr> + + </table> + </div> + + + <br /> + <p>Class specific information for Array (Class 10):</p> + + <p>There are no bit fields defined for the array class. + </p> + + <p>Note that the dimension information defined in the property for this + datatype class is independent of dataspace information for a dataset. + The dimension information here describes the dimensionality of the + information within a data element (or a component of an element, if the + array datatype is nested within another datatype) and the dataspace for a + dataset describes the size and locations of the elements in a dataset. + </p> + + + <div align="center"> + <table class="format"> + <caption> + Array Property Description for Datatype Version 2 + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td>Dimensionality</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4">Dimension #1 Size</td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4">Dimension #n Size</td> + </tr> + + <tr> + <td colspan="4">Permutation Index #1</td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4">Permutation Index #n</td> + </tr> + + <tr> + <td colspan="4"><br />Base Type<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Dimensionality</p></td> + <td> + <p>This value is the number of dimensions that the array has. + </p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Size</p></td> + <td> + <p>This value is the size of the dimension of the array + as stored in the file. The first dimension stored in + the list of dimensions is the slowest changing dimension + and the last dimension stored is the fastest changing + dimension. + </p> + </td> + </tr> + + <tr> + <td><p>Permutation Index #n</p></td> + <td> + <p>This value is the index permutation used to map + each dimension from the canonical representation to an + alternate axis for each dimension. Currently, dimension + permutations are not supported, and these indices should + be set to the index position minus one. In other words, + the first dimension should be set to 0, the second dimension + should be set to 1, and so on. + </p> + </td> + </tr> + + <tr> + <td><p>Base Type</p></td> + <td> + <p>Each array type is based on some parent type. The + information for that parent type is described recursively by + this field. + </p> + </td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Array Property Description for Datatype Version 3 + </caption> + + <tr> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + <th width="25%">Byte</th> + </tr> + + <tr> + <td>Dimensionality</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Dimension #1 Size</td> + </tr> + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + <tr> + <td colspan="4">Dimension #n Size</td> + </tr> + + <tr> + <td colspan="4"><br />Base Type<br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Dimensionality</p></td> + <td> + <p>This value is the number of dimensions that the array has. + </p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Size</p></td> + <td> + <p>This value is the size of the dimension of the array + as stored in the file. The first dimension stored in + the list of dimensions is the slowest changing dimension + and the last dimension stored is the fastest changing + dimension. + </p> + </td> + </tr> + + <tr> + <td><p>Base Type</p></td> + <td> + <p>Each array type is based on some parent type. The + information for that parent type is described recursively by + this field. + </p> + </td> + </tr> + + </table> + </div> + + + +<br /> +<h4><a name="OldFillValueMessage">IV.A.2.e. The Data Storage - +Fill Value (Old) Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Fill Value + (old)</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0004</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td><p>The fill value message stores a single data value which + is returned to the application when an uninitialized data element + is read from a dataset. The fill value is interpreted with the + same datatype as the dataset. If no fill value message is present + then a fill value of all zero bytes is assumed.</p> + <p>This fill value message is deprecated in favor of the + “new” fill value message (Message Type 0x0005) and + is only written to the file for forward compatibility with + versions of the HDF5 Library before the 1.6.0 version. + Additionally, it only appears for datasets with a user-defined + fill value (as opposed to the library default fill value or an + explicitly set “undefined” fill value).</p> + </td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Fill Value Message (Old) + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="4">Size</td> + </tr> + + <tr> + <td colspan="4"><br />Fill Value <em>(optional, variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Size</p></td> + <td> + <p>This is the size of the Fill Value field in bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Fill Value</p></td> + <td> + <p>The fill value. The bytes of the fill value are interpreted + using the same datatype as for the dataset. + </p> + </td> + </tr> + </table> + </div> + + +<br /> +<h4><a name="FillValueMessage">IV.A.2.f. The Data Storage - +Fill Value Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Fill + Value</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0005</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Required for dataset objects; + may not be repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>The fill value message stores a single data value which is + returned to the application when an uninitialized data element + is read from a dataset. The fill value is interpreted with the + same datatype as the dataset.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Fill Value Message - Versions 1 & 2 + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Space Allocation Time</td> + <td>Fill Value Write Time</td> + <td>Fill Value Defined</td> + </tr> + + <tr> + <td colspan="4">Size <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Fill Value <em>(optional, variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number information is used for changes in the + format of the fill value message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Never used + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>Initial version of this message. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>In this version, the Size and Fill Value fields are + only present if the Fill Value Defined field is set + to 1. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>This version packs the other fields in the message + more efficiently than version 2. + </td> + </tr> + </table></p> + </p> + </td> + </tr> + + <tr> + <td><p>Space Allocation Time</p></td> + <td> + <p>When the storage space for the dataset’s raw data will be + allocated. The allowed values are: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Not used. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>Early allocation. Storage space for the entire dataset + should be allocated in the file when the dataset is + created. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>Late allocation. Storage space for the entire dataset + should not be allocated until the dataset is written + to. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>Incremental allocation. Storage space for the + dataset should not be allocated until the portion + of the dataset is written to. This is currently + used in conjunction with chunked data storage for + datasets. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Fill Value Write Time</p></td> + <td> + <p>At the time that storage space for the dataset’s raw data is + allocated, this value indicates whether the fill value should + be written to the raw data storage elements. The allowed values + are: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>On allocation. The fill value is always written to + the raw data storage when the storage space is allocated. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>Never. The fill value should never be written to + the raw data storage. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>Fill value written if set by user. The fill value + will be written to the raw data storage when the storage + space is allocated only if the user explicitly set + the fill value. If the fill value is the library + default or is undefined, it will not be written to + the raw data storage. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Fill Value Defined</p></td> + <td> + <p>This value indicates if a fill value is defined for this + dataset. If this value is 0, the fill value is undefined. + If this value is 1, a fill value is defined for this dataset. + For version 2 or later of the fill value message, this value + controls the presence of the Size and Fill Value fields. + </p> + </td> + </tr> + + <tr> + <td><p>Size</p></td> + <td> + <p>This is the size of the Fill Value field in bytes. This field + is not present if the Version field is greater than 1, + and the Fill Value Defined field is set to 0. + </p> + </td> + </tr> + + <tr> + <td><p>Fill Value</p></td> + <td> + <p>The fill value. The bytes of the fill value are interpreted + using the same datatype as for the dataset. This field is + not present if the Version field is greater than 1, + and the Fill Value Defined field is set to 0. + </p> + </td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Fill Value Message - Version 3 + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Flags</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Size <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Fill Value <em>(optional, variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number information is used for changes in the + format of the fill value message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Never used + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>Initial version of this message. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>In this version, the Size and Fill Value fields are + only present if the Fill Value Defined field is set + to 1. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>This version packs the other fields in the message + more efficiently than version 2. + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td> + <p>When the storage space for the dataset’s raw data will be + allocated. The allowed values are: + <table class="list"> + <tr> + <th width="20%" align="center">Bits</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0-1</code></td> + <td>Space Allocation Time, with the same + values as versions 1 and 2 of the message. + </td> + </tr> + <tr> + <td align="center"><code>2-3</code></td> + <td>Fill Value Write Time, with the same + values as versions 1 and 2 of the message. + </td> + </tr> + <tr> + <td align="center"><code>4</code></td> + <td>Fill Value Undefined, indicating that the fill + value has been marked as “undefined” for this dataset. + Bits 4 and 5 cannot both be set. + </td> + </tr> + <tr> + <td align="center"><code>5</code></td> + <td>Fill Value Defined, with the same values as + versions 1 and 2 of the message. + Bits 4 and 5 cannot both be set. + </td> + </tr> + <tr> + <td align="center"><code>6-7</code></td> + <td>Reserved (zero). + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Size</p></td> + <td> + <p>This is the size of the Fill Value field in bytes. This field + is not present if the Version field is greater than 1, + and the Fill Value Defined flag is set to 0. + </p> + </td> + </tr> + + <tr> + <td><p>Fill Value</p></td> + <td> + <p>The fill value. The bytes of the fill value are interpreted + using the same datatype as for the dataset. This field is + not present if the Version field is greater than 1, + and the Fill Value Defined flag is set to 0. + </p> + </td> + </tr> + </table> + </div> + + +<br /> +<h4><a name="LinkMessage">IV.A.2.g. The Link Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Link</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0006</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies </td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may be + repeated. </td></tr> + <tr><td><b>Description:</b></td> + <td><p>This message encodes the information for a link in a + group’s object header, when the group is storing its links + “compactly”, or in the group’s fractal heap, + when the group is storing its links “densely”.</p> + <p>A group is storing its links compactly when the fractal heap + address in the <em><a href="#LinkInfoMessage">Link Info + Message</a></em> is set to the “undefined address” + value.</p></td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Link Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Flags</td> + <td>Link type <em>(optional)</em></td> + <td bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4"><br />Creation Order <em>(8 bytes, optional)</em><br /><br /></td> + </tr> + <tr> + <td>Link Name Character Set <em>(optional)</em></td> + <td>Length of Link Name (variable size)</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4">Link Name (variable size)</td> + </tr> + <tr> + <td colspan="4"><br />Link Information (variable size)<br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This document describes version 1.</p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>This field contains information about the link and controls + the presence of other fields below. + <table class="list"> + <tr> + <th width="20%" align="center">Bits</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0-1</code></td> + <td>Determines the size of the <em>Length of Link Name</em> + field. + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>The size of the <em>Length of Link Name</em> + field is 1 byte. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>The size of the <em>Length of Link Name</em> + field is 2 bytes. + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>The size of the <em>Length of Link Name</em> + field is 4 bytes. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>The size of the <em>Length of Link Name</em> + field is 8 bytes. + </td> + </tr> + </table> + </td> + </tr> + <tr> + <td align="center"><code>2</code></td> + <td>Creation Order Field Present: if set, the <em>Creation + Order</em> field is present. If not set, creation order + information is not stored for links in this group. + </td> + </tr> + <tr> + <td align="center"><code>3</code></td> + <td>Link Type Field Present: if set, the link is not + a hard link and the <em>Link Type</em> field is present. + If not set, the link is a hard link. + </td> + </tr> + <tr> + <td align="center"><code>4</code></td> + <td>Link Name Character Set Field Present: if set, the + link name is not represented with the ASCII character + set and the <em>Link Name Character Set</em> field is + present. If not set, the link name is represented with + the ASCII character set. + </td> + </tr> + <tr> + <td align="center"><code>5-7</code></td> + <td>Reserved (zero). + </td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Link type</p></td> + <td><p>This is the link class type and can be one of the following + values: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>A hard link (should never be stored in the file) + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>A soft link. + </td> + </tr> + <tr> + <td align="center"><code>2-63</code></td> + <td>Reserved for future HDF5 internal use. + </td> + </tr> + <tr> + <td align="center"><code>64</code></td> + <td>An external link. + </td> + </tr> + <tr> + <td align="center"><code>65-255</code></td> + <td>Reserved, but available for user-defined link types. + </td> + </tr> + </table></p> + + <p>This field is present if bit 3 of <em>Flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Creation Order</p></td> + <td><p>This 64-bit value is an index of the link’s creation time within + the group. Values start at 0 when the group is created an increment + by one for each link added to the group. Removing a link from a + group does not change existing links’ creation order field. + </p> + <p>This field is present if bit 2 of <em>Flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Link Name Character Set</p></td> + <td><p>This is the character set for encoding the link’s name: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>ASCII character set encoding (this should never be stored + in the file) + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>UTF-8 character set encoding + </td> + </tr> + </table></p> + + <p>This field is present if bit 4 of <em>Flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Length of link name</p></td> + <td><p>This is the length of the link’s name. The size of this field + depends on bits 0 and 1 of <em>Flags</em>.</p> + </td> + </tr> + + <tr> + <td><p>Link name</p></td> + <td><p>This is the name of the link, non-NULL terminated.</p> + </td> + </tr> + + <tr> + <td><p>Link information</p></td> + <td><p>The format of this field depends on the <em>link type</em>.</p> + <p>For <b>hard</b> links, the field is formatted as follows: + + <table class="list"> + <tr> + <td width="20%"><i>Size of Offsets</i> bytes:</td> + <td width="80%">The address of the object header for the object that the + link points to. + </td> + </tr> + </table> + </p> + + <p> + For <b>soft</b> links, the field is formatted as follows: + + <table class="list"> + <tr> + <td width="20%">Bytes 1-2:</td> + <td width="80%">Length of soft link value.</td> + </tr> + <tr> + <td><em>Length of soft link value</em> bytes:</td> + <td>A non-NULL-terminated string storing the value of the + soft link. + </td> + </tr> + </table> + </p> + + <p> + For <b>external</b> links, the field is formatted as follows: + + <table class="list"> + <tr> + <td width="20%">Bytes 1-2:</td> + <td width="80%">Length of external link value.</td> + </tr> + <tr> + <td><em>Length of external link value</em> bytes:</td> + <td>The first byte contains the version number in the + upper 4 bits and flags in the lower 4 bits for the external + link. Both version and flags are defined to be zero in + this document. The remaining bytes consist of two + NULL-terminated strings, with no padding between them. + The first string is the name of the HDF5 file containing + the object linked to and the second string is the full path + to the object linked to, within the HDF5 file’s + group hierarchy. + </td> + </tr> + </table> + </p> + + <p> + For <b>user-defined</b> links, the field is formatted as follows: + + <table class="list"> + <tr> + <td width="20%">Bytes 1-2:</td> + <td width="80%">Length of user-defined data.</td> + </tr> + <tr> + <td><em>Length of user-defined link value</em> bytes:</td> + <td>The data supplied for the user-defined link type.</td> + </tr> + </table> + </p> + + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="ExternalFileListMessage">IV.A.2.h. The Data Storage - +External Data Files Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> External + Data Files</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0007</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>The external data storage message indicates that the data + for an object is stored outside the HDF5 file. The filename of + the object is stored as a Universal Resource Location (URL) of + the actual filename containing the data. An external file list + record also contains the byte offset of the start of the data + within the file and the amount of space reserved in the file + for that data.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + External File List Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="2">Allocated Slots</td> + <td colspan="2">Used Slots</td> + </tr> + + <tr> + <td colspan="4"><br />Heap Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Slot Definitions...<br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number information is used for changes in the format of + External Data Storage Message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + <tr> + <td align="center"><code>0</code></td> + <td>Never used.</td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>The current version used by the library.</td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Allocated Slots</p></td> + <td> + <p>The total number of slots allocated in the message. Its value must be at least as + large as the value contained in the Used Slots field. (The current library simply + uses the number of Used Slots for this message)</p> + </td> + </tr> + + <tr> + <td><p>Used Slots</p></td> + <td> + <p>The number of initial slots which contains valid information.</p> + </td> + </tr> + + <tr> + <td><p>Heap Address</p></td> + <td> + <p>This is the address of a local heap which contains the names for the external + files (The local heap information can be found in Disk Format Level 1D in this + document). The name at offset zero in the heap is always the empty string.</p> + </td> + </tr> + + <tr> + <td><p>Slot Definitions</p></td> + <td> + <p>The slot definitions are stored in order according to the array addresses they + represent.</p> + </td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + External File List Slot + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="4"><br />Name Offset in Local Heap<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Offset in External Data File<sup>L</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Data Size in External File<sup>L</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Name Offset in Local Heap</p></td> + <td> + <p>The byte offset within the local name heap for the name + of the file. File names are stored as a URL which has a + protocol name, a host name, a port number, and a file + name: + <code><em>protocol</em>:<em>port</em>//<em>host</em>/<em>file</em></code>. + If the protocol is omitted then “file:” is assumed. If + the port number is omitted then a default port for that + protocol is used. If both the protocol and the port + number are omitted then the colon can also be omitted. If + the double slash and host name are omitted then + “localhost” is assumed. The file name is the only + mandatory part, and if the leading slash is missing then + it is relative to the application’s current working + directory (the use of relative names is not + recommended). + </p> + </td> + </tr> + + <tr> + <td><p>Offset in External Data File</p></td> + <td> + <p>This is the byte offset to the start of the data in the + specified file. For files that contain data for a single + dataset this will usually be zero.</p> + </td> + </tr> + + <tr> + <td><p>Data Size in External File</p></td> + <td> + <p>This is the total number of bytes reserved in the + specified file for raw data storage. For a file that + contains exactly one complete dataset which is not + extendable, the size will usually be the exact size of the + dataset. However, by making the size larger one allows + HDF5 to extend the dataset. The size can be set to a value + larger than the entire file since HDF5 will read zeroes + past the end of the file without failing.</p> + </td> + </tr> + </table> + </div> + + +<br /> +<h4><a name="LayoutMessage">IV.A.2.i. The Data Storage - Layout +Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Data Storage - + Layout</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0008</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Required for datasets; may not + be repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>Data layout describes how the elements of a multi-dimensional + array are stored in the HDF5 file. Three types of data layout + are supported: + <ol> + <li>Contiguous: The array is stored in one contiguous area of + the file. This layout requires that the size of the array be + constant: data manipulations such as chunking, compression, + checksums, or encryption are not permitted. The message stores + the total storage size of the array. The offset of an element + from the beginning of the storage area is computed as in a C + array.</li> + <li>Chunked: The array domain is regularly decomposed into + chunks, and each chunk is allocated and stored separately. This + layout supports arbitrary element traversals, compression, + encryption, and checksums. (these features are described + in other messages). The message stores the size of a chunk + instead of the size of the entire array; the storage size of + the entire array can be calculated by traversing the B-tree + that stores the chunk addresses.</li> + <li>Compact: The array is stored in one contiguous block, as + part of this object header message.</li> + </ol></td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Data Layout Message (Versions 1 and 2) + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Dimensionality</td> + <td>Layout Class</td> + <td>Reserved <em>(zero)</em></td> + </tr> + + <tr> + <td colspan="4">Reserved <em>(zero)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Data Address<sup>O</sup> <em>(optional)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Dimension 0 Size</td> + </tr> + + <tr> + <td colspan="4">Dimension 1 Size</td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">Dimension #n Size</td> + </tr> + + <tr> + <td colspan="4">Dataset Element Size <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4">Compact Data Size <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="4"><br />Compact Data... <em>(variable size, optional)</em><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number information is used for changes in the format of the data + layout message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Never used.</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Used by version 1.4 and before of the library to encode layout information. + Data space is always allocated when the data set is created.</td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Used by version 1.6.x of the library to encode layout information. + Data space is allocated only when it is necessary.</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Dimensionality</p></td> + <td><p>An array has a fixed dimensionality. This field + specifies the number of dimension size fields later in the + message. The value stored for chunked storage is 1 greater than + the number of dimensions in the dataset’s dataspace. + For example, 2 is stored for a 1 dimensional dataset. + </p> + </td> + </tr> + + <tr> + <td><p>Layout Class</p></td> + <td><p>The layout class specifies the type of storage for the data + and how the other fields of the layout message are to be + interpreted. + + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Compact Storage + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Contiguous Storage + </td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Chunked Storage + </td> + </tr> + </table> + </p> + </td> + </tr> + + <tr> + <td><p>Data Address</p></td> + <td><p>For contiguous storage, this is the address of the raw + data in the file. For chunked storage this is the address + of the v1 B-tree that is used to look up the addresses of the + chunks. This field is not present for compact storage. + If the version for this message is greater than 1, the address + may have the “undefined address” value, to indicate that + storage has not yet been allocated for this array.</p> + </td> + </tr> + + <tr> + <td><p>Dimension #n Size</p></td> + <td><p>For contiguous and compact storage the dimensions define + the entire size of the array while for chunked storage they define + the size of a single chunk. In all cases, they are in units of + array elements (not bytes). The first dimension stored in the list + of dimensions is the slowest changing dimension and the last + dimension stored is the fastest changing dimension. + </p> + </td> + </tr> + + <tr> + <td><p>Dataset Element Size</p></td> + <td><p>The size of a dataset element, in bytes. This field is only + present for chunked storage. + </p> + </td> + </tr> + + <tr> + <td><p>Compact Data Size</p></td> + <td><p>This field is only present for compact data storage. + It contains the size of the raw data for the dataset array, in + bytes.</p> + </td> + </tr> + + <tr> + <td><p>Compact Data</p></td> + <td><p>This field is only present for compact data storage. + It contains the raw data for the dataset array.</p> + </td> + </tr> + </table> + </div> + + <br /> + <p>Version 3 of this message re-structured the format into specific + properties that are required for each layout class.</p> + + + <div align="center"> + <table class="format"> + <caption> + <b>Data Layout Message (Version 3)</b> + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Layout Class</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Properties <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td> + <p>The version number information is used for changes in the format of layout message + and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>Used by the version 1.6.3 and later of the library to store properties + for each layout class.</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Layout Class</p></td> + <td><p>The layout class specifies the type of storage for the data + and how the other fields of the layout message are to be + interpreted. + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Compact Storage + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Contiguous Storage + </td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Chunked Storage + </td> + </tr> + </table> + </p> + </td> + </tr> + + <tr> + <td><p>Properties</p></td> + <td><p>This variable-sized field encodes information specific to each + layout class and is described below. If there is no property + information specified for a layout class, the size of this field + is zero bytes.</p></td> + </tr> + </table> + </div> + + <br /> + <p>Class-specific information for compact layout (Class 0): (Note: The dimensionality information + is in the Dataspace message)</p> + + + <div align="center"> + <table class="format"> + <caption> + Compact Storage Property Description + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="2">Size</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Raw Data... <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Size</p></td> + <td><p>This field contains the size of the raw data for the dataset + array, in bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Raw Data</p></td> + <td><p>This field contains the raw data for the dataset array.</p></td> + </tr> + </table> + </div> + + + <br /> + <p>Class-specific information for contiguous layout (Class 1): (Note: The dimensionality information + is in the Dataspace message)</p> + + + <div align="center"> + <table class="format"> + <caption> + Contiguous Storage Property Description + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="4"><br />Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Size<sup>L</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Address</p></td> + <td><p>This is the address of the raw data in the file. + The address may have the “undefined address” value, to indicate + that storage has not yet been allocated for this array.</p></td> + </tr> + + <tr> + <td><p>Size</p></td> + <td><p>This field contains the size allocated to store the raw data, + in bytes. + </p> + </td> + </tr> + </table> + </div> + + + <br /> + <p>Class-specific information for chunked layout (Class 2):</p> + + + <div align="center"> + <table class="format"> + <caption> + Chunked Storage Property Description + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Dimensionality</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Dimension 0 Size</td> + </tr> + + <tr> + <td colspan="4">Dimension 1 Size</td> + </tr> + + <tr> + <td colspan="4">...</td> + </tr> + + <tr> + <td colspan="4">Dimension #n Size</td> + </tr> + + <tr> + <td colspan="4">Dataset Element Size</td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Dimensionality</p></td> + <td><p>A chunk has a fixed dimensionality. This field specifies + the number of dimension size fields later in the message.</p></td> + </tr> + + <tr> + <td><p>Address</p></td> + <td><p>This is the address of the v1 B-tree that is used to look up the + addresses of the chunks that actually store portions of the array + data. The address may have the “undefined address” value, to + indicate that storage has not yet been allocated for this array.</p></td> + </tr> + + <tr> + <td><p>Dimension #n Size</p></td> + <td><p>These values define the dimension size of a single chunk, in + units of array elements (not bytes). The first dimension stored in + the list of dimensions is the slowest changing dimension and the + last dimension stored is the fastest changing dimension. + </p> + </td> + </tr> + + <tr> + <td><p>Dataset Element Size</p></td> + <td><p>The size of a dataset element, in bytes. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="BogusMessage">IV.A.2.j. The Bogus Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Bogus</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0009</td></tr> + <tr><td colspan="2"><b>Length:</b> 4 bytes</td></tr> + <tr><td colspan="2"><b>Status:</b> For testing only; should never + be stored in a valid file.</td></tr> + <tr><td><b>Description:</b></td> + <td>This message is used for testing the HDF5 Library’s + response to an “unknown” message type and should + never be encountered in a valid HDF5 file.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Bogus Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="4">Bogus Value</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Bogus Value</p></td> + <td> + <p>This value should always be: <code>0xdeadbeef</code>.</p> + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="GroupInfoMessage">IV.A.2.k. The Group Info Message +</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Group Info</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x000A</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td><p>This message stores information for the constants defining + a “new style” group’s behavior. Constant + information will be stored in this message and variable + information will be stored in the + <a href="#LinkInfoMessage">Link Info</a> message.</p> + <p>Note: the “estimated entry” information below is + used when determining the size of the object header for the + group when it is created.</p></td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Group Info Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Flags</td> + <td colspan="2">Link Phase Change: Maximum Compact Value <em>(optional)</em></td> + </tr> + <tr> + <td colspan="2">Link Phase Change: Minimum Dense Value <em>(optional)</em></td> + <td colspan="2">Estimated Number of Entries <em>(optional)</em></td> + </tr> + <tr> + <td colspan="2">Estimated Link Name Length of Entries <em>(optional)</em></td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This document describes version 0.</p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>This is the group information flag with the following definition: + + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, link phase change values are stored. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>If set, the estimated entry information is non-default + and is stored. + </td> + </tr> + <tr> + <td align="center"><code>2-7</code></td> + <td>Reserved</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Link Phase Change: Maximum Compact Value</p></td> + <td><p>The is the maximum number of links to store “compactly” (in + the group’s object header).</p> + <p>This field is present if bit 0 of <em>Flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Link Phase Change: Minimum Dense Value</p></td> + <td><p>This is the minimum number of links to store “densely” (in + the group’s fractal heap). The fractal heap’s address is + located in the <a href="#LinkInfoMessage">Link Info</a> + message.</p> + <p>This field is present if bit 0 of <em>Flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Estimated Number of Entries</p></td> + <td><p>This is the estimated number of entries in groups.</p> + <p>If this field is not present, the default value of <code>4</code> + will be used for the estimated number of group entries.</p> + <p>This field is present if bit 1 of <em>Flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Estimated Link Name Length of Entries</p></td> + <td><p>This is the estimated length of entry name.</p> + <p>If this field is not present, the default value of <code>8</code> + will be used for the estimated link name length of group entries.</p> + <p>This field is present if bit 1 of <em>Flags</em> is set.</p> + </td> + </tr> + + </table> + </div> + </p> + +<br /> +<h4><a name="FilterMessage">IV.A.2.l. The Data Storage - Filter +Pipeline Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> + Data Storage - Filter Pipeline</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x000B</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td><p>This message describes the filter pipeline which should + be applied to the data stream by providing filter identification + numbers, flags, a name, and client data.</p> + <p>This message may be present in the object headers of both + dataset and group objects. For datasets, it specifies the + filters to apply to raw data. For groups, it specifies the + filters to apply to the group’s fractal heap. Currently, + only datasets using chunked data storage use the filter + pipeline on their raw data.</p></td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Filter Pipeline Message - Version 1 + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Number of Filters</td> + <td colspan="2">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4"><br />Filter Description List <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This table + describes version 1.</p></td> + </tr> + + <tr> + <td><p>Number of Filters</p></td> + <td><p>The total number of filters described in this + message. The maximum possible number of filters in a + message is 32.</p></td> + </tr> + + <tr> + <td><p>Filter Description List</p></td> + <td><p>A description of each filter. A filter description + appears in the next table.</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Filter Description + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="2">Filter Identification Value</td> + <td colspan="2">Name Length</td> + </tr> + + <tr> + <td colspan="2">Flags</td> + <td colspan="2">Number Client Data Values</td> + </tr> + + <tr> + <td colspan="4"><br />Name <em>(variable size, optional)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Client Data <em>(variable size, optional)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4">Padding <em>(variable size, optional)</em></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Filter Identification Value</p></td> + <td> + <p> + This value, often referred to as a filter identifier, + is designed to be a unique identifier for the filter. + Values from zero through 32,767 are reserved for filters + supported by The HDF Group in the HDF5 Library and for + filters requested and supported by third parties. + Filters supported by The HDF Group are documented immediately + below. Information on 3rd-party filters can be found at + The HDF Group’s + <a href="http://www.hdfgroup.org/services/contributions.html"> + Contributions</a> page.</p> + + <p> + To request a filter identifier, please contact + The HDF Group’s Help Desk at + <img src="Graphics/help.png" valign="middle" height="14" + alt="The HDF Group Help Desk">. + You will be asked to provide the following information:</p> + <ol> + <li>Contact information for the developer requesting the + new identifier</li> + <li>A short description of the new filter</li> + <li>Links to any relevant information, including licensing + information</li> + </ol> + <p> + Values from 32768 to 65535 are reserved for non-distributed uses + (for example, internal company usage) or for application usage + when testing a feature. The HDF Group does not track or document + the use of the filters with identifiers from this range.</p> + + <p> + The filters currently in library version 1.8.0 are + listed below: + + <table class="list"> + <tr> + <th width="20%" align="center">Identification</th> + <th width="15%" align="left">Name</th> + <th width="65%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>N/A</td> + <td>Reserved</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>deflate</td> + <td>GZIP deflate compression</td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>shuffle</td> + <td>Data element shuffling</td> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>fletcher32</td> + <td>Fletcher32 checksum</td> + </tr> + + <tr> + <td align="center"><code>4</code></td> + <td>szip</td> + <td>SZIP compression</td> + </tr> + + <tr> + <td align="center"><code>5</code></td> + <td>nbit</td> + <td>N-bit packing</td> + </tr> + + <tr> + <td align="center"><code>6</code></td> + <td>scaleoffset</td> + <td>Scale and offset encoded values</td> + </tr> + </table> + </p></td> + </tr> + + <tr> + <td><p>Name Length</p></td> + <td><p>Each filter has an optional null-terminated ASCII name + and this field holds the length of the name including the + null termination padded with nulls to be a multiple of + eight. If the filter has no name then a value of zero is + stored in this field.</p></td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>The flags indicate certain properties for a filter. The + bit values defined so far are: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set then the filter is an optional filter. + During output, if an optional filter fails it will be + silently skipped in the pipeline.</td> + </tr> + + <tr> + <td align="center"><code>1-15</code></td> + <td>Reserved (zero)</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Number of Client Data Values</p></td> + <td><p>Each filter can store integer values to control + how the filter operates. The number of entries in the + <em>Client Data</em> array is stored in this field.</p></td> + </tr> + + <tr> + <td><p>Name</p></td> + <td><p>If the <em>Name Length</em> field is non-zero then it will + contain the size of this field, padded to a multiple of eight. This + field contains a null-terminated, ASCII character + string to serve as a comment/name for the filter.</p></td> + </tr> + + <tr> + <td><p>Client Data</p></td> + <td><p>This is an array of four-byte integers which will be + passed to the filter function. The <em>Client Data Number</em> of + Values determines the number of elements in the array.</p></td> + </tr> + + <tr> + <td><p>Padding</p></td> + <td><p>Four bytes of zeroes are added to the message at this + point if the Client Data Number of Values field contains + an odd number.</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Filter Pipeline Message - Version 2 + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Number of Filters</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Filter Description List <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This table + describes version 2.</p></td> + </tr> + + <tr> + <td><p>Number of Filters</p></td> + <td><p>The total number of filters described in this + message. The maximum possible number of filters in a + message is 32.</p></td> + </tr> + + <tr> + <td><p>Filter Description List</p></td> + <td><p>A description of each filter. A filter description + appears in the next table.</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Filter Description + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="2">Filter Identification Value</td> + <td colspan="2">Name Length <em>(optional)</em></td> + </tr> + + <tr> + <td colspan="2">Flags</td> + <td colspan="2">Number Client Data Values</td> + </tr> + + <tr> + <td colspan="4"><br />Name <em>(variable size, optional)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Client Data <em>(variable size, optional)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Filter Identification Value</p></td> + <td> + <p> + This value, often referred to as a filter identifier, + is designed to be a unique identifier for the filter. + Values from zero through 32,767 are reserved for filters + supported by The HDF Group in the HDF5 Library and for + filters requested and supported by third parties. + Filters supported by The HDF Group are documented immediately + below. Information on 3rd-party filters can be found at + The HDF Group’s + <a href="http://www.hdfgroup.org/services/contributions.html"> + Contributions</a> page.</p> + + <p> + To request a filter identifier, please contact + The HDF Group’s Help Desk at + <img src="Graphics/help.png" valign="middle" height="14" + alt="The HDF Group Help Desk">. + You will be asked to provide the following information:</p> + <ol> + <li>Contact information for the developer requesting the + new identifier</li> + <li>A short description of the new filter</li> + <li>Links to any relevant information, including licensing + information</li> + </ol> + <p> + Values from 32768 to 65535 are reserved for non-distributed uses + (for example, internal company usage) or for application usage + when testing a feature. The HDF Group does not track or document + the use of the filters with identifiers from this range.</p> + + <p> + The filters currently in library version 1.8.0 are + listed below: + + <table class="list"> + <tr> + <th width="20%" align="center">Identification</th> + <th width="15%" align="left">Name</th> + <th width="65%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>N/A</td> + <td>Reserved</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>deflate</td> + <td>GZIP deflate compression</td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>shuffle</td> + <td>Data element shuffling</td> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>fletcher32</td> + <td>Fletcher32 checksum</td> + </tr> + + <tr> + <td align="center"><code>4</code></td> + <td>szip</td> + <td>SZIP compression</td> + </tr> + + <tr> + <td align="center"><code>5</code></td> + <td>nbit</td> + <td>N-bit packing</td> + </tr> + + <tr> + <td align="center"><code>6</code></td> + <td>scaleoffset</td> + <td>Scale and offset encoded values</td> + </tr> + </table> + </p></td> + </tr> + + <tr> + <td><p>Name Length</p></td> + <td><p>Each filter has an optional null-terminated ASCII name + and this field holds the length of the name including the + null termination padded with nulls to be a multiple of + eight. If the filter has no name then a value of zero is + stored in this field.</p> + <p>Filters with IDs less than 256 (in other words, filters + that are defined in this format documentation) do not store + the <em>Name Length</em> or <em>Name</em> fields. + </p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>The flags indicate certain properties for a filter. The + bit values defined so far are: + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set then the filter is an optional filter. + During output, if an optional filter fails it will be + silently skipped in the pipeline.</td> + </tr> + + <tr> + <td align="center"><code>1-15</code></td> + <td>Reserved (zero)</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Number of Client Data Values</p></td> + <td><p>Each filter can store integer values to control + how the filter operates. The number of entries in the + <em>Client Data</em> array is stored in this field.</p></td> + </tr> + + <tr> + <td><p>Name</p></td> + <td><p>If the <em>Name Length</em> field is non-zero then it will + contain the size of this field, <em>not</em> padded to a multiple + of eight. This field contains a <em>non-</em>null-terminated, + ASCII character string to serve as a comment/name for the filter. + </p> + <p>Filters that are defined in this format documentation + such as deflate and shuffle do not store the <em>Name + Length</em> or <em>Name</em> fields. + </p> + </td> + </tr> + + <tr> + <td><p>Client Data</p></td> + <td><p>This is an array of four-byte integers which will be + passed to the filter function. The Client Data Number of + Values</em> determines the number of elements in the array.</p> + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="AttributeMessage">IV.A.2.m. The Attribute Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Attribute</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x000C</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td><p>The <em>Attribute</em> message is used to store objects + in the HDF5 file which are used as attributes, or + “metadata” about the current object. An attribute + is a small dataset; it has a name, a datatype, a dataspace, and + raw data. Since attributes are stored in the object header, they + should be relatively small (in other words, less than 64KB). + They can be associated with any type of object which has an + object header (groups, datasets, or committed (named) + datatypes).</p> + <p>In 1.8.x versions of the library, attributes can be larger + than 64KB. See the + <a href="UG/HDF5_Users_Guide-Responsive%20HTML5/index.html#t=HDF5_Users_Guide%2FAttributes%2FHDF5_Attributes.htm%3Frhtocid%3Dtoc8.2_1%23TOC_8_5_Special_Issuesbc-13"> + “Special Issues”</a> section of the Attributes chapter + in the <cite>HDF5 User’s Guide</cite> for more information.</p> + <p>Note: Attributes on an object must have unique names: + the HDF5 Library currently enforces this by causing the + creation of an attribute with a duplicate name to fail. + Attributes on different objects may have the same name, + however.</p></td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Attribute Message (Version 1) + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Reserved (zero)</td> + <td colspan="2">Name Size</td> + </tr> + + <tr> + <td colspan="2">Datatype Size</td> + <td colspan="2">Dataspace Size</td> + </tr> + + <tr> + <td colspan="4"><br />Name <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Datatype <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Dataspace <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Data <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number information is used for changes in the format of the + attribute message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Never used.</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Used by the library before version 1.6 to encode attribute message. + This version does not support shared datatypes.</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Name Size</p></td> + <td><p>The length of the attribute name in bytes including the + null terminator. Note that the <em>Name</em> field below may + contain additional padding not represented by this + field.</p></td> + </tr> + + <tr> + <td><p>Datatype Size</p></td> + <td><p>The length of the datatype description in the <em>Datatype</em> + field below. Note that the <em>Datatype</em> field may contain + additional padding not represented by this field.</p></td> + </tr> + + <tr> + <td><p>Dataspace Size</p></td> + <td><p>The length of the dataspace description in the <em>Dataspace</em> + field below. Note that the <em>Dataspace</em> field may contain + additional padding not represented by this field.</p></td> + </tr> + + <tr> + <td><p>Name</p></td> + <td><p>The null-terminated attribute name. This field is + padded with additional null characters to make it a + multiple of eight bytes.</p></td> + </tr> + + <tr> + <td><p>Datatype</p></td> + <td><p>The datatype description follows the same format as + described for the datatype object header message. This + field is padded with additional zero bytes to make it a + multiple of eight bytes.</p></td> + </tr> + + <tr> + <td><p>Dataspace</p></td> + <td><p>The dataspace description follows the same format as + described for the dataspace object header message. This + field is padded with additional zero bytes to make it a + multiple of eight bytes.</p></td> + </tr> + + <tr> + <td><p>Data</p></td> + <td><p>The raw data for the attribute. The size is determined + from the datatype and dataspace descriptions. This + field is <em>not</em> padded with additional bytes.</p></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Attribute Message (Version 2) + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Flags</td> + <td colspan="2">Name Size</td> + </tr> + + <tr> + <td colspan="2">Datatype Size</td> + <td colspan="2">Dataspace Size</td> + </tr> + + <tr> + <td colspan="4"><br />Name <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Datatype <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Dataspace <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Data <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number information is used for changes in the + format of the attribute message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>Used by the library of version 1.6.x and after to encode + attribute messages. + This version supports shared datatypes. The fields of + name, datatype, and dataspace are not padded with + additional bytes of zero. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>This bit field contains extra information about + interpreting the attribute message: + + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, datatype is shared.</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>If set, dataspace is shared.</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Name Size</p></td> + <td><p>The length of the attribute name in bytes including the + null terminator.</p></td> + </tr> + + <tr> + <td><p>Datatype Size</p></td> + <td><p>The length of the datatype description in the <em>Datatype</em> + field below.</p></td> + </tr> + + <tr> + <td><p>Dataspace Size</p></td> + <td><p>The length of the dataspace description in the <em>Dataspace</em> + field below.</p></td> + </tr> + + <tr> + <td><p>Name</p></td> + <td><p>The null-terminated attribute name. This field is <em>not</em> + padded with additional bytes.</p></td> + </tr> + + <tr> + <td><p>Datatype</p></td> + <td><p>The datatype description follows the same format as + described for the datatype object header message. + </p> + <p>If the + <em>Flag</em> field indicates this attribute’s datatype is + shared, this field will contain a “shared message” encoding + instead of the datatype encoding. + </p> + <p>This field is <em>not</em> padded with additional bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Dataspace</p></td> + <td><p>The dataspace description follows the same format as + described for the dataspace object header message. + </p> + <p>If the + <em>Flag</em> field indicates this attribute’s dataspace is + shared, this field will contain a “shared message” encoding + instead of the dataspace encoding. + </p> + <p>This field is <em>not</em> padded with additional bytes.</p> + </td> + </tr> + + <tr> + <td><p>Data</p></td> + <td><p>The raw data for the attribute. The size is determined + from the datatype and dataspace descriptions. + </p> + <p>This field is <em>not</em> padded with additional zero bytes. + </p> + </td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="format"> + <caption> + Attribute Message (Version 3) + </caption> + + <tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Flags</td> + <td colspan="2">Name Size</td> + </tr> + + <tr> + <td colspan="2">Datatype Size</td> + <td colspan="2">Dataspace Size</td> + </tr> + + <tr> + <td>Name Character Set Encoding</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Name <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Datatype <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Dataspace <em>(variable size)</em><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Data <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number information is used for changes in the + format of the attribute message and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>Used by the library of version 1.8.x and after to + encode attribute messages. + This version supports attributes with non-ASCII names. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>This bit field contains extra information about + interpreting the attribute message: + + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, datatype is shared.</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>If set, dataspace is shared.</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Name Size</p></td> + <td><p>The length of the attribute name in bytes including the + null terminator.</p></td> + </tr> + + <tr> + <td><p>Datatype Size</p></td> + <td><p>The length of the datatype description in the <em>Datatype</em> + field below.</p></td> + </tr> + + <tr> + <td><p>Dataspace Size</p></td> + <td><p>The length of the dataspace description in the <em>Dataspace</em> + field below.</p></td> + </tr> + + <tr> + <td><p>Name Character Set Encoding</p></td> + <td><p>The character set encoding for the attribute’s name: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>ASCII character set encoding + </td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>UTF-8 character set encoding + </td> + </tr> + </table> + </p> + </td> + </tr> + + <tr> + <td><p>Name</p></td> + <td><p>The null-terminated attribute name. This field is <em>not</em> + padded with additional bytes.</p></td> + </tr> + + <tr> + <td><p>Datatype</p></td> + <td><p>The datatype description follows the same format as + described for the datatype object header message. + </p> + <p>If the + <em>Flag</em> field indicates this attribute’s datatype is + shared, this field will contain a “shared message” encoding + instead of the datatype encoding. + </p> + <p>This field is <em>not</em> padded with additional bytes. + </p> + </td> + </tr> + + <tr> + <td><p>Dataspace</p></td> + <td><p>The dataspace description follows the same format as + described for the dataspace object header message. + </p> + <p>If the + <em>Flag</em> field indicates this attribute’s dataspace is + shared, this field will contain a “shared message” encoding + instead of the dataspace encoding. + </p> + <p>This field is <em>not</em> padded with additional bytes.</p> + </td> + </tr> + + <tr> + <td><p>Data</p></td> + <td><p>The raw data for the attribute. The size is determined + from the datatype and dataspace descriptions. + </p> + <p>This field is <em>not</em> padded with additional zero bytes. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="CommentMessage">IV.A.2.n. The Object Comment +Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Object + Comment</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x000D</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>The object comment is designed to be a short description of + an object. An object comment is a sequence of non-zero + (<code>\0</code>) ASCII characters with no other formatting + included by the library.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Name Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="4"><br />Comment <em>(variable size)</em><br /><br /></td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Name</p></td> + <td><p>A null terminated ASCII character string.</p></td> + </tr> + </table> + </div> + +<br /> +<h4><a name="OldModificationTimeMessage">IV.A.2.o. The Object +Modification Time (Old) Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Object + Modification Time (Old)</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x000E</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td><p>The object modification date and time is a timestamp + which indicates (using ISO-8601 date and time format) the last + modification of an object. The time is updated when any object + header message changes according to the system clock where the + change was posted. All fields of this message should be + interpreted as coordinated universal time (UTC).</p> + <p>This modification time message is deprecated in favor of + the “new” <a href="#ModificationTimeMessage">Object + Modification Time</a> message and is no longer written to the + file in versions of the HDF5 Library after the 1.6.0 + version.</p></td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Modification Time Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="4">Year</td> + </tr> + + <tr> + <td colspan="2">Month</td> + <td colspan="2">Day of Month</td> + </tr> + + <tr> + <td colspan="2">Hour</td> + <td colspan="2">Minute</td> + </tr> + + <tr> + <td colspan="2">Second</td> + <td colspan="2">Reserved</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Year</p></td> + <td><p>The four-digit year as an ASCII string. For example, + <code>1998</code>. + </p></td> + </tr> + + <tr> + <td><p>Month</p></td> + <td><p>The month number as a two digit ASCII string where + January is <code>01</code> and December is <code>12</code>.</p></td> + </tr> + + <tr> + <td><p>Day of Month</p></td> + <td><p>The day number within the month as a two digit ASCII + string. The first day of the month is <code>01</code>.</p></td> + </tr> + + <tr> + <td><p>Hour</p></td> + <td><p>The hour of the day as a two digit ASCII string where + midnight is <code>00</code> and 11:00pm is <code>23</code>.</p></td> + </tr> + + <tr> + <td><p>Minute</p></td> + <td><p>The minute of the hour as a two digit ASCII string where + the first minute of the hour is <code>00</code> and + the last is <code>59</code>.</p></td> + </tr> + + <tr> + <td><p>Second</p></td> + <td><p>The second of the minute as a two digit ASCII string + where the first second of the minute is <code>00</code> + and the last is <code>59</code>.</p></td> + </tr> + + <tr> + <td><p>Reserved</p></td> + <td><p>This field is reserved and should always be zero.</p></td> + </tr> + </table> + </div> + +<br /> +<h4><a name="SOHMTableMessage">IV.A.2.p. The Shared Message Table +Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Shared Message + Table</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x000F</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>This message is used to locate the table of shared object + header message (SOHM) indexes. Each index consists of information + to find the shared messages from either the heap or object header. + This message is <em>only</em> found in the superblock + extension.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Shared Message Table Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Shared Object Header Message Table Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td>Number of Indices</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This document describes version 0.</p></td> + </tr> + + <tr> + <td><p>Shared Object Header Message Table Address</p></td> + <td><p>This field is the address of the master table for shared + object header message indexes.</p> + </td> + </tr> + + <tr> + <td><p>Number of Indices</p></td> + <td><p>This field is the number of indices in the master table. + </p></td> + </tr> + + </table> + </div> + +<br /> +<h4><a name="ContinuationMessage">IV.A.2.q. The Object Header +Continuation Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Object Header + Continuation</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0010</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>The object header continuation is the location in the file + of a block containing more header messages for the current data + object. This can be used when header blocks become too large or + are likely to change over time.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Object Header Continuation Message + </caption> + + <tr> + <th width=25%>byte</th> + <th width=25%>byte</th> + <th width=25%>byte</th> + <th width=25%>byte</th> + </tr> + + <tr> + <td colspan="4"><br />Offset<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Length<sup>L</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Offset</p></td> + <td><p>This value is the address in the file where the + header continuation block is located.</p></td> + </tr> + + <tr> + <td><p>Length</p></td> + <td><p>This value is the length in bytes of the header continuation + block in the file.</p></td> + </tr> + </table> + </div> + <br /> + + <p>The format of the header continuation block that this message points + to depends on the version of the object header that the message is + contained within. + </p> + + <p> + Continuation blocks for version 1 object headers have no special + formatting information; they are merely a list of object header + message info sequences (type, size, flags, reserved bytes and data + for each message sequence). See the description + of <a href="#V1ObjectHeaderPrefix">Version 1 Data Object Header Prefix.</a> + </p> + + <p>Continuation blocks for version 2 object headers <em>do</em> have + special formatting information as described here + (see also the description of + <a href="#V2ObjectHeaderPrefix">Version 2 Data Object Header Prefix.</a>): + </p> + <div align="center"> + <table class="format"> + <caption> + Version 2 Object Header Continuation Block + </caption> + + <tr> + <th>byte</th> + <th>byte</th> + <th>byte</th> + <th>byte</th> + </tr> + + <tr> + <td colspan="4">Signature</td> + </tr> + <tr> + <td>Header Message Type #1</td> + <td colspan="2">Size of Header Message Data #1</td> + <td>Header Message #1 Flags</td> + </tr> + + <tr> + <td colspan="2">Header Message #1 Creation Order <em>(optional)</em></td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Header Message Data #1<br /><br /></td> + </tr> + + <tr> + <td colspan="4">.<br />.<br />.<br /></td> + </tr> + + <tr> + <td>Header Message Type #n</td> + <td colspan="2">Size of Header Message Data #n</td> + <td>Header Message #n Flags</td> + </tr> + + <tr> + <td colspan="2">Header Message #n Creation Order <em>(optional)</em></td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br />Header Message Data #n<br /><br /></td> + </tr> + + <tr> + <td colspan="4">Gap <em>(optional, variable size)</em></td> + </tr> + + <tr> + <td colspan="4">Checksum</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Signature</p></td> + <td> + <p>The ASCII character string “<code>OCHK</code>” + is used to indicate the + beginning of an object header continuation block. This gives file + consistency checking utilities a better chance of reconstructing a + damaged file. + </p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Type</p></td> + <td> + <p>Same format as version 1 of the object header, described above. + </p></td> + </tr> + + <tr> + <td><p>Size of Header Message #n Data</p></td> + <td> + <p>Same format as version 1 of the object header, described above. + </p></td> + </tr> + + <tr> + <td><p>Header Message #n Flags</p></td> + <td> + <p>Same format as version 1 of the object header, described above. + </p></td> + </tr> + + <tr> + <td><p>Header Message #n Creation Order</p></td> + <td> + <p>This field stores the order that a message of a given type + was created in.</p> + <p>This field is present if bit 2 of <em>flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Header Message #n Data</p></td> + <td> + <p>Same format as version 1 of the object header, described above. + </p></td> + </tr> + + <tr> + <td><p>Gap</p></td> + <td> + <p>A gap in an object header chunk is inferred by the end of the + messages for the chunk before the beginning of the chunk’s + checksum. Gaps are always smaller than the size of an + object header message prefix (message type + message size + + message flags).</p> + <p>Gaps are formed when a message (typically an attribute message) + in an earlier chunk is deleted and a message from a later + chunk that does not quite fit into the free space is moved + into the earlier chunk.</p> + </td> + </tr> + + <tr> + <td><p>Checksum</p></td> + <td> + <p>This is the checksum for the object header chunk. + </p> + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="SymbolTableMessage">IV.A.2.r. The Symbol Table +Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Symbol Table + Message</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0011</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Required for + “old style” groups; may not be repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>Each “old style” group has a v1 B-tree and a + local heap for storing symbol table entries, which are located + with this message.</td></tr> + <tr><td colspan="2"><b>Format of data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + <b>Symbol Table Message</b> + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td colspan="4"><br />v1 B-tree Address<sup>O</sup><br /><br /></td> + </tr> + + <tr> + <td colspan="4"><br />Local Heap Address<sup>O</sup><br /><br /></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>v1 B-tree Address</p></td> + <td><p>This value is the address of the v1 B-tree containing the + symbol table entries for the group.</p></td> + </tr> + + <tr> + <td><p>Local Heap Address</p></td> + <td><p>This value is the address of the local heap containing + the link names for the symbol table entries for the group.</p></td> + </tr> + </table> + </div> + +<br /> +<h4><a name="ModificationTimeMessage">IV.A.2.s. The Object +Modification Time Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Object + Modification Time</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0012</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>The object modification time is a timestamp which indicates + the time of the last modification of an object. The time is + updated when any object header message changes according to + the system clock where the change was posted.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Modification Time Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td colspan="3">Reserved (zero)</td> + </tr> + + <tr> + <td colspan="4">Seconds After UNIX Epoch</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number is used for changes in the format of Object Modification Time + and is described here: + <table class="list"> + <tr> + <th width="20%" align="center">Version</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>Never used.</td> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>Used by Version 1.6.1 and after of the library to encode time. In + this version, the time is the seconds after Epoch.</td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Seconds After UNIX Epoch</p></td> + <td><p>A 32-bit unsigned integer value that stores the number of + seconds since 0 hours, 0 minutes, 0 seconds, January 1, 1970, + Coordinated Universal Time.</p></td> + </tr> + </table> + </div> + +<br /> +<h4><a name="BtreeKValuesMessage">IV.A.2.t. The B-tree +‘K’ Values Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> B-tree + ‘K’ Values</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0013</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>This message retrieves non-default ‘K’ values + for internal and leaf nodes of a group or indexed storage v1 + B-trees. This message is <em>only</em> found in the superblock + extension.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + B-tree ‘K’ Values Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td colspan="2">Indexed Storage Internal Node K</td> + <td bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="2">Group Internal Node K</td> + <td colspan="2">Group Leaf Node K</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This document describes + version 0.</p> + </td> + </tr> + + <tr> + <td><p>Indexed Storage Internal Node K</p></td> + <td><p>This is the node ‘K’ value for each internal node of an + indexed storage v1 B-tree. See the description of this field + in version 0 and 1 of the superblock as well the section on + v1 B-trees. + </p> + </td> + </tr> + + <tr> + <td><p>Group Internal Node K</p></td> + <td><p>This is the node ‘K’ value for each internal node of a group + v1 B-tree. See the description of this field in version 0 and + 1 of the superblock as well as the section on v1 B-trees. + </p> + </td> + </tr> + + <tr> + <td><p>Group Leaf Node K</p></td> + <td><p>This is the node ‘K’ value for each leaf node of a group v1 + B-tree. See the description of this field in version 0 and 1 + of the superblock as well as the section on v1 B-trees. + </p> + </td> + </tr> + + </table> + </div> + +<br /> +<h4><a name="DrvInfoMessage">IV.A.2.u. The Driver Info +Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Driver + Info</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0014</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + + <tr><td> + <b>Description:</b></td> + <td>This message contains information needed by the file driver + to reopen a file. This message is <em>only</em> found in the + superblock extension: see the <a href="#SuperblockExt"> + “Disk Format: Level 0C - Superblock Extension”</a> + section for more information. For more information on the fields + in the driver info message, see the <a href="#DriverInfo"> + “Disk Format : Level 0B - File Driver Info”</a> + section; those who use the multi and family file drivers will + find this section particularly helpful.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Driver Info Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + <tr> + <td colspan="4"><br />Driver Identification</td> + </tr> + + <tr> + <td colspan="2">Driver Information Size</td> + <td colspan="2" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4"><br /><br />Driver Information <em>(variable size)</em><br /><br /><br /></td> + </tr> + + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This document describes + version 0.</p> + </td> + </tr> + + <tr> + <td><p>Driver Identification</p></td> + <td><p>This is an eight-byte ASCII string without null termination which + identifies the driver. + </p> + </td> + </tr> + + <tr> + <td><p>Driver Information Size</p></td> + <td><p>The size in bytes of the <em>Driver Information</em> field of this + message.</p> + </td> + </tr> + + <tr> + <td><p>Driver Information</p></td> + <td><p>Driver information is stored in a format defined by the file driver.</p> + </td> + </tr> + </table> + </div> + +<br /> +<h4><a name="AinfoMessage">IV.A.2.v. The Attribute Info +Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Attribute + Info</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0015</td></tr> + <tr><td colspan="2"><b>Length:</b> Varies</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>This message stores information about the attributes on an + object, such as the maximum creation index for the attributes + created and the location of the attribute storage when the + attributes are stored “densely”.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Attribute Info Message + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Flags</td> + <td colspan="2">Maximum Creation Index <em>(optional)</em></td> + </tr> + <tr> + <td colspan="4"><br />Fractal Heap Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Attribute Name v2 B-tree Address<sup>O</sup><br /><br /></td> + </tr> + <tr> + <td colspan="4"><br />Attribute Creation Order v2 B-tree Address<sup>O</sup> <em>(optional)</em><br /><br /></td> + </tr> + + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This document describes + version 0.</p> + </td> + </tr> + + <tr> + <td><p>Flags</p></td> + <td><p>This is the attribute index information flag with the + following definition: + + <table class="list"> + <tr> + <th width="20%" align="center">Bit</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>0</code></td> + <td>If set, creation order for attributes is tracked. + </td> + </tr> + <tr> + <td align="center"><code>1</code></td> + <td>If set, creation order for attributes is indexed. + </td> + </tr> + <tr> + <td align="center"><code>2-7</code></td> + <td>Reserved</td> + </tr> + </table></p> + + </td> + </tr> + + <tr> + <td><p>Maximum Creation Index</p></td> + <td><p>The is the maximum creation order index value for the + attributes on the object.</p> + <p>This field is present if bit 0 of <em>Flags</em> is set.</p> + </td> + </tr> + + <tr> + <td><p>Fractal Heap Address</p></td> + <td><p>This is the address of the fractal heap to store dense + attributes.</p> + </td> + </tr> + + <tr> + <td><p>Attribute Name v2 B-tree Address</p></td> + <td><p>This is the address of the version 2 B-tree to index the + names of densely stored attributes.</p> + </td> + </tr> + + <tr> + <td><p>Attribute Creation Order v2 B-tree Address</p></td> + <td><p>This is the address of the version 2 B-tree to index the + creation order of densely stored attributes.</p> + <p>This field is present if bit 1 of <em>Flags</em> is set.</p> + </td> + </tr> + + </table> + </div> + +<br /> +<h4><a name="RefCountMessage">IV.A.2.w. The Object Reference +Count Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> Object Reference + Count</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0016</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td><b>Description:</b></td> + <td>This message stores the number of hard links (in groups or + objects) pointing to an object: in other words, its + <em>reference count</em>.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + Object Reference Count + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td colspan="3" bgcolor="#DDDDDD"><em>This space inserted only to align table nicely</em></td> + </tr> + + <tr> + <td colspan="4">Reference count</td> + </tr> + </table> + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>The version number for this message. This document describes + version 0.</p> + </td> + </tr> + + <tr> + <td><p>Reference Count</p></td> + <td><p>The unsigned 32-bit integer is the reference count for the + object. This message is only present in “version 2” + (or later) object headers, and if not present those object + header versions, the reference count for the object is assumed + to be 1.</p> + </td> + </tr> + + </table> + </div> + +<br /> +<h4><a name="FsinfoMessage">IV.A.2.x. The File Space Info +Message</a></h4> + + <!-- start msgdesc table --> + <center> + <table class="msgdesc"> + <tr><td colspan="2"><b>Header Message Name:</b> File Space + Info</td></tr> + <tr><td colspan="2"><b>Header Message Type:</b> 0x0018</td></tr> + <tr><td colspan="2"><b>Length:</b> Fixed</td></tr> + <tr><td colspan="2"><b>Status:</b> Optional; may not be + repeated.</td></tr> + <tr><td> + <b>Description:</b></td> + <td>This message stores the file space management strategy (see + description below) that the library uses in handling file space + request for the file. It also contains the free-space section + threshold used by the library’s free-space managers for + the file. If the strategy is 1, this message also contains the + addresses of the file’s free-space managers which track + free space for each type of file space allocation. There are + six basic types of file space allocation: superblock, B-tree, + raw data, global heap, local heap, and object header. See the + description of <a href="#FreeSpaceManager">Free-space + Manager</a> as well the description of allocation types in + <a href="#AppendixB">Appendix B</a>.</td></tr> + <tr><td colspan="2"><b>Format of Data:</b> See the tables + below.</td></tr> + </table></center> + <!-- end msgdesc table --> + + <div align="center"> + <table class="format"> + <caption> + File Space Info + </caption> + + <tr> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr> + <td>Version</td> + <td>Strategy</td> + <td colspan="2">Threshold<sup>L</sup></td> + </tr> + <tr> + <td colspan="4">Super-block Free-space Manager Address<sup>O</sup></td> + </tr> + <tr> + <td colspan="4">B-tree Free-space Manager Address<sup>O</sup></td> + </tr> + <tr> + <td colspan="4">Raw Data Free-space Manager Address<sup>O</sup></td> + </tr> + <tr> + <td colspan="4">Global Heap Free-space Manager Address<sup>O</sup></td> + </tr> + <tr> + <td colspan="4">Local Heap Free-space Manager Address<sup>O</sup></td> + </tr> + <tr> + <td colspan="4">Object Header Free-space Manager Address<sup>O</sup></td> + </tr> + </table> + + <table class="note"> + <tr> + <td width="60%"> </td> + <td width="40%"> + (Items marked with an ‘O’ in the above table are of the size + specified in “Size of Offsets” field in the superblock.) + </td></tr> + <tr> + <td> </td> + <td> + (Items marked with an ‘L’ in the above table are of the size + specified in “Size of Lengths” field in the superblock.) + </td></tr> + </table> + + </div> + + <br /> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Field Name</th> + <th>Description</th> + </tr> + + <tr> + <td><p>Version</p></td> + <td><p>This is the version number of this message. This document describes + version 0.</p> + </td> + </tr> + + <tr> + <td><p>Strategy</p></td> + <td><p>This is the file space management strategy for the file. + There are four types of strategies: + <table class="list"> + <tr> + <th width="20%" align="center">Value</th> + <th width="80%" align="left">Description</th> + </tr> + + <tr> + <td align="center"><code>1</code></td> + <td>With this strategy, the HDF5 Library’s free-space managers track the + free space that results from the manipulation of HDF5 objects + in the HDF5 file. The free space information is saved when the + file is closed, and reloaded when the file is reopened. + <br /> + When space is needed for file metadata or raw data, + the HDF5 Library first requests space from the library’s free-space + managers. If the request is not satisfied, the library requests space + from the aggregators. If the request is still not satisfied, + the library requests space from the virtual file driver. + That is, the library will use all of the mechanisms for allocating + space. + </td> + </tr> + + <tr> + <td align="center"><code>2</code></td> + <td>This is the HDF5 Library’s default file space management strategy. + With this strategy, the library’s free-space managers track the free space + that results from the manipulation of HDF5 objects in the HDF5 file. + The free space information is NOT saved when the file is closed and + the free space that exists upon file closing becomes unaccounted + space in the file. + <br /> + As with strategy #1, the library will try all of the mechanisms + for allocating space. When space is needed for file metadata or + raw data, the library first requests space from the free-space + managers. If the request is not satisfied, the library requests + space from the aggregators. If the request is still not satisfied, + the library requests space from the virtual file driver. + </td> + </tr> + + <tr> + <td align="center"><code>3</code></td> + <td>With this strategy, the HDF5 Library does not track free space that results + from the manipulation of HDF5 objects in the HDF5 file and + the free space becomes unaccounted space in the file. + <br /> + When space is needed for file metadata or raw data, + the library first requests space from the aggregators. + If the request is not satisfied, the library requests space from + the virtual file driver. + </td> + </tr> + <tr> + <td align="center"><code>4</code></td> + <td>With this strategy, the HDF5 Library does not track free space that results + from the manipulation of HDF5 objects in the HDF5 file and + the free space becomes unaccounted space in the file. + <br /> + When space is needed for file metadata or raw data, + the library requests space from the virtual file driver. + </td> + </tr> + </table></p> + </td> + </tr> + + <tr> + <td><p>Threshold</p></td> + <td><p>This is the free-space section threshold. + The library’s free-space managers will track only + free-space sections with size greater than or equal to + <em>threshold</em>. The default is to track free-space + sections of all sizes.</p> + </td> + </tr> + <tr> + <td><p>Superblock Free-space Manager Address</p></td> + <td><p>This is the address of the free-space manager for + H5FD_MEM_SUPER allocation type. + </p> + </td> + </tr> + + <tr> + <td><p>B-tree Free-space Manager Address</p></td> + <td><p>This is the address of the free-space manager for + H5FD_MEM_BTREE allocation type. + </p> + </td> + </tr> + + <tr> + <td><p>Raw Data Free-space Manager Address</p></td> + <td><p>This is the address of the free-space manager for + H5FD_MEM_DRAW allocation type. + </p> + </td> + </tr> + + <tr> + <td><p>Global Heap Free-space Manager Address</p></td> + <td><p>This is the address of the free-space manager for + H5FD_MEM_GHEAP allocation type. + </p> + </td> + </tr> + + <tr> + <td><p>Local Heap Free-space Manager Address</p></td> + <td><p>This is the address of the free-space manager for + H5FD_MEM_LHEAP allocation type. + </p> + </td> + </tr> + + <tr> + <td><p>Object Header Free-space Manager Address</p></td> + <td><p>This is the address of the free-space manager for + H5FD_MEM_OHDR allocation type. + </p> + </td> + </tr> + </table> + </div> + <br /> + + +<br /> +<h3><a name="DataStorage"> +IV.B. Disk Format: Level 2B - Data Object Data Storage</a></h3> + +<p>The data for an object is stored separately from its header + information in the file and may not actually be located in the HDF5 file + itself if the header indicates that the data is stored externally. The + information for each record in the object is stored according to the + dimensionality of the object (indicated in the dataspace header message). + Multi-dimensional array data is stored in C order; in other words, the + “last” dimension changes fastest.</p> + +<p>Data whose elements are composed of atomic datatypes are stored in IEEE + format, unless they are specifically defined as being stored in a different + machine format with the architecture-type information from the datatype + header message. This means that each architecture will need to [potentially] + byte-swap data values into the internal representation for that particular + machine.</p> + +<p> Data with a variable-length datatype is stored in the global heap + of the HDF5 file. Global heap identifiers are stored in the + data object storage.</p> + +<p>Data whose elements are composed of reference datatypes are stored in + several different ways depending on the particular reference type involved. + Object pointers are just stored as the offset of the object header being + pointed to with the size of the pointer being the same number of bytes as + offsets in the file.</p> + +<p>Dataset region references are stored as a heap-ID which points to +the following information within the file-heap: an offset of the object +pointed to, number-type information (same format as header message), +dimensionality information (same format as header message), sub-set start +and end information (in other words, a coordinate location for each), +and field start and end names (in other words, a [pointer to the] string +indicating the first field included and a [pointer to the] string name +for the last field). </p> + +<p>Data of a compound datatype is stored as a contiguous stream of the items + in the structure, with each item formatted according to its datatype.</p> + + + +<br /> +<br /> +<hr /> +<h2><a name="AppendixA"> +V. Appendix A: Definitions</a></h2> + +<p>Definitions of various terms used in this document are included in +this section.</p> + + <div align="center"> + <table class="glossary"> + <tr> + <th width="20%">Term</th> + <th>Definition</th> + </tr> + + <tr> + <td>Undefined Address</td> + <td>The <a name="UndefinedAddress">undefined + address</a> for a file is a file address with all bits + set: in other words, <code>0xffff...ff</code>.</td> + </tr> + + <tr> + <td>Unlimited Size</td> + <td>The <a name="UnlimitedDim">unlimited size</a> + for a size is a value with all bits set: in other words, + <code>0xffff...ff</code>.</td> + </tr> + + </table> + </div> + + + +<br /> +<br /> +<hr /> +<h2><a name="AppendixB"> +VI. Appendix B: File Memory Allocation Types</a></h2> + +<p>There are six basic types of file memory allocation as follows: +</p> + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Basic Allocation Type</th> + <th>Description</th> + </tr> + + <tr> + <td>H5FD_MEM_SUPER</td> + <td>File memory allocated for <em>Superblock.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_BTREE</td> + <td>File memory allocated for <em>B-tree.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_DRAW</td> + <td>File memory allocated for raw data.</td> + </tr> + + <tr> + <td>H5FD_MEM_GHEAP</td> + <td>File memory allocated for <em>Global Heap.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_LHEAP</td> + <td>File memory allocated for <em>Local Heap.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_OHDR</td> + <td>File memory allocated for <em>Object Header.</em></td> + </tr> + </table> + </div> + +<p>There are other file memory allocation types that are mapped to the +above six basic allocation types because they are similar in nature. +The mapping is listed in the following table: +</p> + + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Basic Allocation Type</th> + <th>Mapping of Allocation Types to Basic Allocation Types</th> + </tr> + + <tr> + <td>H5FD_MEM_SUPER</td> + <td><em>none</em></td> + </tr> + + <tr> + <td>H5FD_MEM_BTREE</td> + <td>H5FD_MEM_SOHM_INDEX</td> + </tr> + + <tr> + <td>H5FD_MEM_DRAW</td> + <td>H5FD_MEM_FHEAP_HUGE_OBJ</td> + </tr> + + <tr> + <td>H5FD_MEM_GHEAP</td> + <td><em>none</em></td> + </tr> + + <tr> + <td>H5FD_MEM_LHEAP</td> + <td>H5FD_MEM_FHEAP_DBLOCK, H5FD_MEM_FSPACE_SINFO</td> + </tr> + + <tr> + <td>H5FD_MEM_OHDR</td> + <td>H5FD_MEM_FHEAP_HDR, H5FD_MEM_FHEAP_IBLOCK, H5FD_MEM_FSPACE_HDR, H5FD_MEM_SOHM_TABLE</td> + </tr> + </table> + </div> + +<p>Allocation types that are mapped to basic allocation types are described below: +</p> + + <div align="center"> + <table class="desc"> + <tr> + <th width="30%">Allocation Type</th> + <th>Description</th> + </tr> + + <tr> + <td>H5FD_MEM_FHEAP_HDR</td> + <td>File memory allocated for <em>Fractal Heap Header.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_FHEAP_DBLOCK</td> + <td>File memory allocated for <em>Fractal Heap Direct Blocks.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_FHEAP_IBLOCK</td> + <td>File memory allocated for <em>Fractal Heap Indirect Blocks.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_FHEAP_HUGE_OBJ</td> + <td>File memory allocated for huge objects in the fractal heap.</td> + </tr> + + <tr> + <td>H5FD_MEM_FSPACE_HDR</td> + <td>File memory allocated for <em>Free-space Manager Header.</em></td> + </tr> + + <tr> + <td>H5FD_MEM_FSPACE_SINFO</td> + <td>File memory allocated for <em>Free-space Section List</em> of the free-space manager.</td> + </tr> + <tr> + <td>H5FD_MEM_SOHM_TABLE</td> + <td>File memory allocated for <em>Shared Object Header Message Table.</em></td> + </tr> + <tr> + <td>H5FD_MEM_SOHM_INDEX</td> + <td>File memory allocated for <em>Shared Message Record List.</em></td> + </tr> + </table> + </div> +</body> +</html> |