1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
|
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Robb Matzke <matzke@llnl.gov>
* Monday, July 26, 1999
*/
#ifndef _H5FDprivate_H
#define _H5FDprivate_H
/* Include package's public header */
#include "H5FDpublic.h"
/* Private headers needed by this file */
#include "H5Pprivate.h" /* Property lists */
/*
* The MPI drivers are needed because there are
* places where we check for things that aren't handled by these drivers.
*/
#include "H5FDmpi.h" /* MPI-based file drivers */
/**************************/
/* Library Private Macros */
/**************************/
/* Length of filename buffer */
#define H5FD_MAX_FILENAME_LEN 1024
/*
* VFD SWMR
*/
/* Metadata file header */
#define H5FD_MD_HEADER_MAGIC "VHDR" /* Header magic */
#define H5FD_SIZEOF_CHKSUM 4 /* Size of checksum */
/* Size of the header in the metadata file */
#define H5FD_MD_HEADER_SIZE \
( \
H5_SIZEOF_MAGIC /* Signature */ \
+ 4 /* Page size */ \
+ 8 /* Tick number */ \
+ 8 /* Index offset */ \
+ 8 /* Index length number */ \
+ H5FD_SIZEOF_CHKSUM /* Metadata header checksum */ \
)
/* Size of an index entry in the metadata file */
#define H5FD_MD_INDEX_ENTRY_SIZE \
( \
4 /* HDF5 file page offset */ \
+ 4 /* Metadata file page offset */ \
+ 4 /* Length */ \
+ H5FD_SIZEOF_CHKSUM /* Index entry checksum */ \
)
/* Metadata file index magic */
#define H5FD_MD_INDEX_MAGIC "VIDX" /* Index magic */
/* Size of the metadata file index */
#define H5FD_MD_INDEX_SIZE(N) /* N is number of entries in index */ \
( \
H5_SIZEOF_MAGIC /* Signature */ \
+ 8 /* Tick num */ \
+ 4 /* Number of entries */ \
+ (N * H5FD_MD_INDEX_ENTRY_SIZE) /* Index entries */ \
+ H5FD_SIZEOF_CHKSUM /* Metadata header checksum */ \
)
/* Retries for metadata file */
#define H5FD_VFD_SWMR_MD_FILE_RETRY_MAX 50 /* Maximum retries when opening the MD file */
#define H5FD_VFD_SWMR_MD_LOAD_RETRY_MAX 20 /* Maximum retries when trying to load the MD file header and index */
#define H5FD_VFD_SWMR_MD_HEADER_RETRY_MAX 40 /* Maximum retries when deserializing the MD file header */
#define H5FD_VFD_SWMR_MD_INDEX_RETRY_MAX 5 /* Maximum retries when deserializing the MD file index */
/*
* fs_page_size: Size of pages in both the HDF5 file and the metadata file IN BYTES
* tick_num: Sequence number of the current tick.
* Initialized to zero on file creation/open, and incremented by the
* VFD SWMR writer at the end of each tick.
* index_offset: The offset of the current metadata file index in the metadata file
* IN BYTES.
* index_length: The length of the current metadata file index IN BYTES.
*/
typedef struct H5FD_vfd_swmr_md_header {
uint32_t fs_page_size;
uint64_t tick_num;
uint64_t index_offset;
uint64_t index_length;
} H5FD_vfd_swmr_md_header;
/* Internal representation of metadata file index entry */
/*
* hdf5_page_offset: Unsigned 64-bit value containing the base address of the
* metadata page, or multi page metadata entry in the HDF5
* file IN PAGES.
* To obtain byte offset, multiply this value by the page size.
*
* md_file_page_offset: Unsigned 64-bit value containing the base address of the
* metadata page, or multi page metadata entry in the metadata
* file IN PAGES.
* To obtain byte offset, multiply this value by the page size.
*
* length: The length of the metadata page or multi- page metadata entry
* in BYTES.
*
* chksum: Checksum for the metadata page or multi-page metadata entry.
* For the VFD SWMR writer, this value is undefined until the
* referenced entry has been written to the metadata file.
*
* entry_ptr: Used by the VFD SWMR writer only.
* For the VFD SWMR reader, this field should always be NULL.
* If the referenced metadata page or multi-page metadata
* entry was modified in the current tick, this field points to
* a buffer in the page buffer containing its value.
* This field is used by the metadata file creation/update code
* to access the metadata pages or multi-page metadata entries
* so that their current values can be copied into the metadata
* file. After this copy, this field should be set to NULL.
*
* tick_of_last_change: Number of the last tick in which this index entry was changed.
* Used by the VFD SWMR writer only.
* For the VFD SWMR reader, this field will always be set to 0.
*
* clean: Used by the VFD SWMR writer only.
* Set to TRUE whenever the referenced metadata page or multi-page
* metadata entry is written to the HDF5 file.
* Set to FALSE whenever it is marked dirty in the page buffer.
*
* tick_of_last_flush: Number of the tick in which this entry was last written to the
* HDF5 file or zero if it has never been flusehd.
* Used by the VFD SWMR writer only.
* For the VFD SWMR reader, this field should always be 0.
*
* delayed_flush: If the flush of the referenced metadata page or multi-page
* metadata entry must be delayed, the earliest tick in which
* it may be flushed, or zero if there is no such constraint.
* Used by the VFD SWMR writer only.
*
* is_moved_to_hdf5_file: Set to TRUE iff the entry referenced is in the HDF5 file and
* is therefore about to be removed from the metadata file
*/
typedef struct H5FD_vfd_swmr_idx_entry_t {
uint64_t hdf5_page_offset;
uint64_t md_file_page_offset;
uint32_t length;
uint32_t chksum;
void *entry_ptr;
uint64_t tick_of_last_change;
hbool_t clean;
uint64_t tick_of_last_flush;
uint64_t delayed_flush;
hbool_t moved_to_hdf5_file;
} H5FD_vfd_swmr_idx_entry_t;
/*
* tick_num: Sequence number of the current tick.
* Initialized to zero on file creation/open, and incremented by the
* VFD SWMR writer at the end of each tick.
* num_entries: The number of entires in the index.
* entries: The array of index entries
*/
typedef struct H5FD_vfd_swmr_md_index {
uint64_t tick_num;
uint32_t num_entries;
H5FD_vfd_swmr_idx_entry_t *entries;
} H5FD_vfd_swmr_md_index;
#ifdef H5_HAVE_PARALLEL
/* ======== Temporary data transfer properties ======== */
/* Definitions for memory MPI type property */
#define H5FD_MPI_XFER_MEM_MPI_TYPE_NAME "H5FD_mpi_mem_mpi_type"
/* Definitions for file MPI type property */
#define H5FD_MPI_XFER_FILE_MPI_TYPE_NAME "H5FD_mpi_file_mpi_type"
/* Sub-class the H5FD_class_t to add more specific functions for MPI-based VFDs */
typedef struct H5FD_class_mpi_t {
H5FD_class_t super; /* Superclass information & methods */
int (*get_rank)(const H5FD_t *file); /* Get the MPI rank of a process */
int (*get_size)(const H5FD_t *file); /* Get the MPI size of a communicator */
MPI_Comm (*get_comm)(const H5FD_t *file); /* Get the communicator for a file */
herr_t (*get_mpi_info)(H5FD_t *file, void** mpi_info); /* get MPI_Info for a file */
} H5FD_class_mpi_t;
#endif
/****************************/
/* Library Private Typedefs */
/****************************/
/* File operations */
typedef enum {
OP_UNKNOWN = 0, /* Unknown last file operation */
OP_READ = 1, /* Last file I/O operation was a read */
OP_WRITE = 2 /* Last file I/O operation was a write */
} H5FD_file_op_t;
/* Define structure to hold initial file image and other relevant information */
typedef struct {
void *buffer;
size_t size;
H5FD_file_image_callbacks_t callbacks;
} H5FD_file_image_info_t;
/* Define default file image info */
#define H5FD_DEFAULT_FILE_IMAGE_INFO { \
/* file image buffer */ NULL, \
/* buffer size */ 0, \
{ /* Callbacks */ \
/* image_malloc */ NULL, \
/* image_memcpy */ NULL, \
/* image_realloc */ NULL, \
/* image_free */ NULL, \
/* udata_copy */ NULL, \
/* udata_free */ NULL, \
/* udata */ NULL, \
} \
}
/* Define structure to hold driver ID & info for FAPLs */
typedef struct {
hid_t driver_id; /* Driver's ID */
const void *driver_info; /* Driver info, for open callbacks */
} H5FD_driver_prop_t;
#ifdef H5_HAVE_PARALLEL
/* MPIO-specific file access properties */
typedef struct H5FD_mpio_fapl_t {
MPI_Comm comm; /*communicator */
MPI_Info info; /*file information */
} H5FD_mpio_fapl_t;
#endif /* H5_HAVE_PARALLEL */
/*****************************/
/* Library Private Variables */
/*****************************/
/******************************/
/* Library Private Prototypes */
/******************************/
/* Forward declarations for prototype arguments */
struct H5F_t;
H5_DLL int H5FD_term_interface(void);
H5_DLL herr_t H5FD_locate_signature(H5FD_t *file, haddr_t *sig_addr);
H5_DLL H5FD_class_t *H5FD_get_class(hid_t id);
H5_DLL hsize_t H5FD_sb_size(H5FD_t *file);
H5_DLL herr_t H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf);
H5_DLL herr_t H5FD_sb_load(H5FD_t *file, const char *name, const uint8_t *buf);
H5_DLL void *H5FD_fapl_get(H5FD_t *file);
H5_DLL herr_t H5FD_fapl_close(hid_t driver_id, const void *fapl);
H5_DLL hid_t H5FD_register(const void *cls, size_t size, hbool_t app_ref);
H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id,
haddr_t maxaddr);
H5_DLL herr_t H5FD_close(H5FD_t *file);
H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2);
H5_DLL herr_t H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags/*out*/);
H5_DLL haddr_t H5FD_alloc(H5FD_t *file, H5FD_mem_t type,
struct H5F_t *f, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size);
H5_DLL herr_t H5FD_free(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f,
haddr_t addr, hsize_t size);
H5_DLL htri_t H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f,
haddr_t blk_end, hsize_t extra_requested);
H5_DLL haddr_t H5FD_get_eoa(const H5FD_t *file, H5FD_mem_t type);
H5_DLL herr_t H5FD_set_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr);
H5_DLL haddr_t H5FD_get_eof(const H5FD_t *file, H5FD_mem_t type);
H5_DLL haddr_t H5FD_get_maxaddr(const H5FD_t *file);
H5_DLL herr_t H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags);
H5_DLL herr_t H5FD_set_feature_flags(H5FD_t *file, unsigned long feature_flags);
H5_DLL herr_t H5FD_get_fs_type_map(const H5FD_t *file, H5FD_mem_t *type_map);
H5_DLL herr_t H5FD_read(H5FD_t *file, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/);
H5_DLL herr_t H5FD_write(H5FD_t *file, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf);
H5_DLL herr_t H5FD_flush(H5FD_t *file, hbool_t closing);
H5_DLL herr_t H5FD_truncate(H5FD_t *file, hbool_t closing);
H5_DLL herr_t H5FD_lock(H5FD_t *file, hbool_t rw);
H5_DLL herr_t H5FD_unlock(H5FD_t *file);
H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum);
H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle);
H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr);
H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file);
H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged);
/* Function prototypes for VFD SWMR */
H5_DLL herr_t H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t read_index,
uint64_t *tick_ptr, uint32_t *num_entries_ptr, H5FD_vfd_swmr_idx_entry_t index[]);
/* Function prototypes for MPI based VFDs*/
#ifdef H5_HAVE_PARALLEL
/* General routines */
H5_DLL haddr_t H5FD_mpi_MPIOff_to_haddr(MPI_Offset mpi_off);
H5_DLL herr_t H5FD_mpi_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off/*out*/);
H5_DLL herr_t H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info,
MPI_Comm *comm_new, MPI_Info *info_new);
H5_DLL herr_t H5FD_mpi_comm_info_free(MPI_Comm *comm, MPI_Info *info);
#ifdef NOT_YET
H5_DLL herr_t H5FD_mpio_wait_for_left_neighbor(H5FD_t *file);
H5_DLL herr_t H5FD_mpio_signal_right_neighbor(H5FD_t *file);
#endif /* NOT_YET */
H5_DLL herr_t H5FD_set_mpio_atomicity(H5FD_t *file, hbool_t flag);
H5_DLL herr_t H5FD_get_mpio_atomicity(H5FD_t *file, hbool_t *flag);
/* Driver specific methods */
H5_DLL int H5FD_mpi_get_rank(const H5FD_t *file);
H5_DLL int H5FD_mpi_get_size(const H5FD_t *file);
H5_DLL MPI_Comm H5FD_mpi_get_comm(const H5FD_t *_file);
H5_DLL herr_t H5FD_get_mpi_info(H5FD_t *file, void** file_info);
#endif /* H5_HAVE_PARALLEL */
#endif /* !_H5FDprivate_H */
|