summaryrefslogtreecommitdiffstats
path: root/src/H5FDpublic.h
blob: 0fcfcf373c9047e10aa0ba3bdd782d116baac4a4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Copyright by The HDF Group.                                               *
 * Copyright by the Board of Trustees of the University of Illinois.         *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of HDF5.  The full HDF5 copyright notice, including     *
 * terms governing use, modification, and redistribution, is contained in    *
 * the COPYING file, which can be found at the root of the source code       *
 * distribution tree, or in https://www.hdfgroup.org/licenses.               *
 * If you do not have access to either file, you may request a copy from     *
 * help@hdfgroup.org.                                                        *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

/*
 * Programmer:  Robb Matzke
 *              Monday, July 26, 1999
 */
#ifndef H5FDpublic_H
#define H5FDpublic_H

#include "H5public.h"
#include "H5Fpublic.h" /*for H5F_close_degree_t */

#define H5_HAVE_VFL      1 /*define a convenient app feature test*/
#define H5FD_VFD_DEFAULT 0 /* Default VFL driver value */

/* Types of allocation requests: see H5Fpublic.h  */
typedef enum H5F_mem_t H5FD_mem_t;

/* Map "fractal heap" header blocks to 'ohdr' type file memory, since its
 * a fair amount of work to add a new kind of file memory and they are similar
 * enough to object headers and probably too minor to deserve their own type.
 *
 * Map "fractal heap" indirect blocks to 'ohdr' type file memory, since they
 * are similar to fractal heap header blocks.
 *
 * Map "fractal heap" direct blocks to 'lheap' type file memory, since they
 * will be replacing local heaps.
 *
 * Map "fractal heap" 'huge' objects to 'draw' type file memory, since they
 * represent large objects that are directly stored in the file.
 *
 *      -QAK
 */
#define H5FD_MEM_FHEAP_HDR      H5FD_MEM_OHDR
#define H5FD_MEM_FHEAP_IBLOCK   H5FD_MEM_OHDR
#define H5FD_MEM_FHEAP_DBLOCK   H5FD_MEM_LHEAP
#define H5FD_MEM_FHEAP_HUGE_OBJ H5FD_MEM_DRAW

/* Map "free space" header blocks to 'ohdr' type file memory, since its
 * a fair amount of work to add a new kind of file memory and they are similar
 * enough to object headers and probably too minor to deserve their own type.
 *
 * Map "free space" serialized sections to 'lheap' type file memory, since they
 * are similar enough to local heap info.
 *
 *      -QAK
 */
#define H5FD_MEM_FSPACE_HDR   H5FD_MEM_OHDR
#define H5FD_MEM_FSPACE_SINFO H5FD_MEM_LHEAP

/* Map "shared object header message" master table to 'ohdr' type file memory,
 * since its a fair amount of work to add a new kind of file memory and they are
 * similar enough to object headers and probably too minor to deserve their own
 * type.
 *
 * Map "shared object header message" indices to 'btree' type file memory,
 * since they are similar enough to B-tree nodes.
 *
 *      -QAK
 */
#define H5FD_MEM_SOHM_TABLE H5FD_MEM_OHDR
#define H5FD_MEM_SOHM_INDEX H5FD_MEM_BTREE

/* Map "extensible array" header blocks to 'ohdr' type file memory, since its
 * a fair amount of work to add a new kind of file memory and they are similar
 * enough to object headers and probably too minor to deserve their own type.
 *
 * Map "extensible array" index blocks to 'ohdr' type file memory, since they
 * are similar to extensible array header blocks.
 *
 * Map "extensible array" super blocks to 'btree' type file memory, since they
 * are similar enough to B-tree nodes.
 *
 * Map "extensible array" data blocks & pages to 'lheap' type file memory, since
 * they are similar enough to local heap info.
 *
 *      -QAK
 */
#define H5FD_MEM_EARRAY_HDR       H5FD_MEM_OHDR
#define H5FD_MEM_EARRAY_IBLOCK    H5FD_MEM_OHDR
#define H5FD_MEM_EARRAY_SBLOCK    H5FD_MEM_BTREE
#define H5FD_MEM_EARRAY_DBLOCK    H5FD_MEM_LHEAP
#define H5FD_MEM_EARRAY_DBLK_PAGE H5FD_MEM_LHEAP

/* Map "fixed array" header blocks to 'ohdr' type file memory, since its
 * a fair amount of work to add a new kind of file memory and they are similar
 * enough to object headers and probably too minor to deserve their own type.
 *
 * Map "fixed array" data blocks & pages to 'lheap' type file memory, since
 * they are similar enough to local heap info.
 *
 */
#define H5FD_MEM_FARRAY_HDR       H5FD_MEM_OHDR
#define H5FD_MEM_FARRAY_DBLOCK    H5FD_MEM_LHEAP
#define H5FD_MEM_FARRAY_DBLK_PAGE H5FD_MEM_LHEAP

/*
 * A free-list map which maps all types of allocation requests to a single
 * free list.  This is useful for drivers that don't really care about
 * keeping different requests segregated in the underlying file and which
 * want to make most efficient reuse of freed memory.  The use of the
 * H5FD_MEM_SUPER free list is arbitrary.
 */
#define H5FD_FLMAP_SINGLE                                                                                    \
    {                                                                                                        \
        H5FD_MEM_SUPER,     /*default*/                                                                      \
            H5FD_MEM_SUPER, /*super*/                                                                        \
            H5FD_MEM_SUPER, /*btree*/                                                                        \
            H5FD_MEM_SUPER, /*draw*/                                                                         \
            H5FD_MEM_SUPER, /*gheap*/                                                                        \
            H5FD_MEM_SUPER, /*lheap*/                                                                        \
            H5FD_MEM_SUPER  /*ohdr*/                                                                         \
    }

/*
 * A free-list map which segregates requests into `raw' or `meta' data
 * pools.
 */
#define H5FD_FLMAP_DICHOTOMY                                                                                 \
    {                                                                                                        \
        H5FD_MEM_SUPER,     /*default*/                                                                      \
            H5FD_MEM_SUPER, /*super*/                                                                        \
            H5FD_MEM_SUPER, /*btree*/                                                                        \
            H5FD_MEM_DRAW,  /*draw*/                                                                         \
            H5FD_MEM_DRAW,  /*gheap*/                                                                        \
            H5FD_MEM_SUPER, /*lheap*/                                                                        \
            H5FD_MEM_SUPER  /*ohdr*/                                                                         \
    }

/*
 * The default free list map which causes each request type to use it's own
 * free-list.
 */
#define H5FD_FLMAP_DEFAULT                                                                                   \
    {                                                                                                        \
        H5FD_MEM_DEFAULT,     /*default*/                                                                    \
            H5FD_MEM_DEFAULT, /*super*/                                                                      \
            H5FD_MEM_DEFAULT, /*btree*/                                                                      \
            H5FD_MEM_DEFAULT, /*draw*/                                                                       \
            H5FD_MEM_DEFAULT, /*gheap*/                                                                      \
            H5FD_MEM_DEFAULT, /*lheap*/                                                                      \
            H5FD_MEM_DEFAULT  /*ohdr*/                                                                       \
    }

/* Define VFL driver features that can be enabled on a per-driver basis */
/* These are returned with the 'query' function pointer in H5FD_class_t */
/*
 * Defining H5FD_FEAT_AGGREGATE_METADATA for a VFL driver means that
 * the library will attempt to allocate a larger block for metadata and
 * then sub-allocate each metadata request from that larger block.
 */
#define H5FD_FEAT_AGGREGATE_METADATA 0x00000001
/*
 * Defining H5FD_FEAT_ACCUMULATE_METADATA for a VFL driver means that
 * the library will attempt to cache metadata as it is written to the file
 * and build up a larger block of metadata to eventually pass to the VFL
 * 'write' routine.
 *
 * Distinguish between updating the metadata accumulator on writes and
 * reads.  This is particularly (perhaps only, even) important for MPI-I/O
 * where we guarantee that writes are collective, but reads may not be.
 * If we were to allow the metadata accumulator to be written during a
 * read operation, the application would hang.
 */
#define H5FD_FEAT_ACCUMULATE_METADATA_WRITE 0x00000002
#define H5FD_FEAT_ACCUMULATE_METADATA_READ  0x00000004
#define H5FD_FEAT_ACCUMULATE_METADATA                                                                        \
    (H5FD_FEAT_ACCUMULATE_METADATA_WRITE | H5FD_FEAT_ACCUMULATE_METADATA_READ)
/*
 * Defining H5FD_FEAT_DATA_SIEVE for a VFL driver means that
 * the library will attempt to cache raw data as it is read from/written to
 * a file in a "data seive" buffer.  See Rajeev Thakur's papers:
 *  http://www.mcs.anl.gov/~thakur/papers/romio-coll.ps.gz
 *  http://www.mcs.anl.gov/~thakur/papers/mpio-high-perf.ps.gz
 */
#define H5FD_FEAT_DATA_SIEVE 0x00000008
/*
 * Defining H5FD_FEAT_AGGREGATE_SMALLDATA for a VFL driver means that
 * the library will attempt to allocate a larger block for "small" raw data
 * and then sub-allocate "small" raw data requests from that larger block.
 */
#define H5FD_FEAT_AGGREGATE_SMALLDATA 0x00000010
/*
 * Defining H5FD_FEAT_IGNORE_DRVRINFO for a VFL driver means that
 * the library will ignore the driver info that is encoded in the file
 * for the VFL driver.  (This will cause the driver info to be eliminated
 * from the file when it is flushed/closed, if the file is opened R/W).
 */
#define H5FD_FEAT_IGNORE_DRVRINFO 0x00000020
/*
 * Defining the H5FD_FEAT_DIRTY_DRVRINFO_LOAD for a VFL driver means that
 * the library will mark the driver info dirty when the file is opened
 * R/W.  This will cause the driver info to be re-encoded when the file
 * is flushed/closed.
 */
#define H5FD_FEAT_DIRTY_DRVRINFO_LOAD 0x00000040
/*
 * Defining H5FD_FEAT_POSIX_COMPAT_HANDLE for a VFL driver means that
 * the handle for the VFD (returned with the 'get_handle' callback) is
 * of type 'int' and is compatible with POSIX I/O calls.
 */
#define H5FD_FEAT_POSIX_COMPAT_HANDLE 0x00000080
/*
 * Defining H5FD_FEAT_HAS_MPI for a VFL driver means that
 * the driver makes use of MPI communication and code may retrieve
 * communicator/rank information from it
 */
#define H5FD_FEAT_HAS_MPI 0x00000100
/*
 * Defining the H5FD_FEAT_ALLOCATE_EARLY for a VFL driver will force
 * the library to use the H5D_ALLOC_TIME_EARLY on dataset create
 * instead of the default H5D_ALLOC_TIME_LATE
 */
#define H5FD_FEAT_ALLOCATE_EARLY 0x00000200
/*
 * Defining H5FD_FEAT_ALLOW_FILE_IMAGE for a VFL driver means that
 * the driver is able to use a file image in the fapl as the initial
 * contents of a file.
 */
#define H5FD_FEAT_ALLOW_FILE_IMAGE 0x00000400
/*
 * Defining H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS for a VFL driver
 * means that the driver is able to use callbacks to make a copy of the
 * image to store in memory.
 */
#define H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS 0x00000800
/*
 * Defining H5FD_FEAT_SUPPORTS_SWMR_IO for a VFL driver means that the
 * driver supports the single-writer/multiple-readers I/O pattern.
 */
#define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000
/*
 * Defining H5FD_FEAT_USE_ALLOC_SIZE for a VFL driver
 * means that the library will just pass the allocation size to the
 * the driver's allocation callback which will eventually handle alignment.
 * This is specifically used for the multi/split driver.
 */
#define H5FD_FEAT_USE_ALLOC_SIZE 0x00002000
/*
 * Defining H5FD_FEAT_PAGED_AGGR for a VFL driver
 * means that the driver needs special file space mapping for paged aggregation.
 * This is specifically used for the multi/split driver.
 */
#define H5FD_FEAT_PAGED_AGGR 0x00004000
/*
 * Defining H5FD_FEAT_DEFAULT_VFD_COMPATIBLE for a VFL driver
 * that creates a file which is compatible with the default VFD.
 * Generally, this means that the VFD creates a single file that follows
 * the canonical HDF5 file format.
 * Regarding the Splitter VFD specifically, only drivers with this flag
 * enabled may be used as the Write-Only (W/O) channel driver.
 */
#define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000

/* Forward declaration */
typedef struct H5FD_t H5FD_t;

/* Class information for each file driver */
typedef struct H5FD_class_t {
    const char *       name;
    haddr_t            maxaddr;
    H5F_close_degree_t fc_degree;
    herr_t (*terminate)(void);
    hsize_t (*sb_size)(H5FD_t *file);
    herr_t (*sb_encode)(H5FD_t *file, char *name /*out*/, unsigned char *p /*out*/);
    herr_t (*sb_decode)(H5FD_t *f, const char *name, const unsigned char *p);
    size_t fapl_size;
    void *(*fapl_get)(H5FD_t *file);
    void *(*fapl_copy)(const void *fapl);
    herr_t (*fapl_free)(void *fapl);
    size_t dxpl_size;
    void *(*dxpl_copy)(const void *dxpl);
    herr_t (*dxpl_free)(void *dxpl);
    H5FD_t *(*open)(const char *name, unsigned flags, hid_t fapl, haddr_t maxaddr);
    herr_t (*close)(H5FD_t *file);
    int (*cmp)(const H5FD_t *f1, const H5FD_t *f2);
    herr_t (*query)(const H5FD_t *f1, unsigned long *flags);
    herr_t (*get_type_map)(const H5FD_t *file, H5FD_mem_t *type_map);
    haddr_t (*alloc)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
    herr_t (*free)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size);
    haddr_t (*get_eoa)(const H5FD_t *file, H5FD_mem_t type);
    herr_t (*set_eoa)(H5FD_t *file, H5FD_mem_t type, haddr_t addr);
    haddr_t (*get_eof)(const H5FD_t *file, H5FD_mem_t type);
    herr_t (*get_handle)(H5FD_t *file, hid_t fapl, void **file_handle);
    herr_t (*read)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, void *buffer);
    herr_t (*write)(H5FD_t *file, H5FD_mem_t type, hid_t dxpl, haddr_t addr, size_t size, const void *buffer);
    herr_t (*flush)(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
    herr_t (*truncate)(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
    herr_t (*lock)(H5FD_t *file, hbool_t rw);
    herr_t (*unlock)(H5FD_t *file);
    herr_t (*del)(const char *name, hid_t fapl);
    H5FD_mem_t fl_map[H5FD_MEM_NTYPES];
} H5FD_class_t;

/* A free list is a singly-linked list of address/size pairs. */
typedef struct H5FD_free_t {
    haddr_t             addr;
    hsize_t             size;
    struct H5FD_free_t *next;
} H5FD_free_t;

/*
 * The main datatype for each driver. Public fields common to all drivers
 * are declared here and the driver appends private fields in memory.
 */
struct H5FD_t {
    hid_t               driver_id;     /*driver ID for this file   */
    const H5FD_class_t *cls;           /*constant class info       */
    unsigned long       fileno;        /* File 'serial' number     */
    unsigned            access_flags;  /* File access flags (from create or open) */
    unsigned long       feature_flags; /* VFL Driver feature Flags */
    haddr_t             maxaddr;       /* For this file, overrides class */
    haddr_t             base_addr;     /* Base address for HDF5 data w/in file */

    /* Space allocation management fields */
    hsize_t threshold;  /* Threshold for alignment  */
    hsize_t alignment;  /* Allocation alignment     */
    hbool_t paged_aggr; /* Paged aggregation for file space is enabled or not */
};

/* Define enum for the source of file image callbacks */
typedef enum {
    H5FD_FILE_IMAGE_OP_NO_OP,
    H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
    H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY,
    H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET,
    H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE,
    H5FD_FILE_IMAGE_OP_FILE_OPEN,
    H5FD_FILE_IMAGE_OP_FILE_RESIZE,
    H5FD_FILE_IMAGE_OP_FILE_CLOSE
} H5FD_file_image_op_t;

/* Define structure to hold file image callbacks */
typedef struct {
    void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op, void *udata);
    void *(*image_memcpy)(void *dest, const void *src, size_t size, H5FD_file_image_op_t file_image_op,
                          void *udata);
    void *(*image_realloc)(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *udata);
    herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op, void *udata);
    void *(*udata_copy)(void *udata);
    herr_t (*udata_free)(void *udata);
    void *udata;
} H5FD_file_image_callbacks_t;

#ifdef __cplusplus
extern "C" {
#endif

/* Function prototypes */
H5_DLL hid_t  H5FDregister(const H5FD_class_t *cls);
H5_DLL herr_t H5FDunregister(hid_t driver_id);
H5_DLL H5FD_t *H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr);
H5_DLL herr_t  H5FDclose(H5FD_t *file);
H5_DLL int     H5FDcmp(const H5FD_t *f1, const H5FD_t *f2);
H5_DLL int     H5FDquery(const H5FD_t *f, unsigned long *flags);
H5_DLL haddr_t H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
H5_DLL herr_t  H5FDfree(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size);
H5_DLL haddr_t H5FDget_eoa(H5FD_t *file, H5FD_mem_t type);
H5_DLL herr_t  H5FDset_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t eoa);
H5_DLL haddr_t H5FDget_eof(H5FD_t *file, H5FD_mem_t type);
H5_DLL herr_t  H5FDget_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle);
H5_DLL herr_t  H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
                        void *buf /*out*/);
H5_DLL herr_t  H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
                         const void *buf);
H5_DLL herr_t  H5FDflush(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
H5_DLL herr_t  H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
H5_DLL herr_t  H5FDlock(H5FD_t *file, hbool_t rw);
H5_DLL herr_t  H5FDunlock(H5FD_t *file);
H5_DLL herr_t  H5FDdelete(const char *name, hid_t fapl_id);

/* Allows querying a VFD ID for features before the file is opened */
H5_DLL herr_t H5FDdriver_query(hid_t driver_id, unsigned long *flags /*out*/);

#ifdef __cplusplus
}
#endif
#endif