summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/CMakeLists.txt261
-rw-r--r--src/H5FD.c1054
-rw-r--r--src/H5FDcore.h2
-rw-r--r--src/H5FDhdfs.c1876
-rw-r--r--src/H5FDhdfs.h123
-rw-r--r--src/H5FDprivate.h16
-rw-r--r--src/H5FDpublic.h73
-rw-r--r--src/H5FDros3.c1787
-rw-r--r--src/H5FDros3.h106
-rw-r--r--src/H5FDs3comms.c3593
-rw-r--r--src/H5FDs3comms.h610
-rw-r--r--src/Makefile.am14
-rw-r--r--src/Makefile.in11
-rw-r--r--src/libhdf5.settings.in88
14 files changed, 8901 insertions, 713 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 86deb10..544859f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -4,7 +4,7 @@ project (HDF5_SRC C)
#-----------------------------------------------------------------------------
# List Source Files
#-----------------------------------------------------------------------------
-set (H5_SRCS
+set (H5_SOURCES
${HDF5_SRC_DIR}/H5.c
${HDF5_SRC_DIR}/H5checksum.c
${HDF5_SRC_DIR}/H5dbg.c
@@ -20,9 +20,9 @@ set (H5_HDRS
#${HDF5_SRC_DIR}/H5version.h
#${HDF5_SRC_DIR}/H5overflow.h
)
-IDE_GENERATED_PROPERTIES ("H5" "${H5_HDRS}" "${H5_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5" "${H5_HDRS}" "${H5_SOURCES}" )
-set (H5A_SRCS
+set (H5A_SOURCES
${HDF5_SRC_DIR}/H5A.c
${HDF5_SRC_DIR}/H5Abtree2.c
${HDF5_SRC_DIR}/H5Adense.c
@@ -34,28 +34,28 @@ set (H5A_SRCS
set (H5A_HDRS
${HDF5_SRC_DIR}/H5Apublic.h
)
-IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SOURCES}" )
-set (H5AC_SRCS
+set (H5AC_SOURCES
${HDF5_SRC_DIR}/H5AC.c
)
set (H5AC_HDRS
${HDF5_SRC_DIR}/H5ACpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5AC" "${H5AC_HDRS}" "${H5AC_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5AC" "${H5AC_HDRS}" "${H5AC_SOURCES}" )
-set (H5B_SRCS
+set (H5B_SOURCES
${HDF5_SRC_DIR}/H5B.c
${HDF5_SRC_DIR}/H5Bcache.c
${HDF5_SRC_DIR}/H5Bdbg.c
)
set (H5B_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5B" "${H5B_HDRS}" "${H5B_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5B" "${H5B_HDRS}" "${H5B_SOURCES}" )
-set (H5B2_SRCS
+set (H5B2_SOURCES
${HDF5_SRC_DIR}/H5B2.c
${HDF5_SRC_DIR}/H5B2cache.c
${HDF5_SRC_DIR}/H5B2dbg.c
@@ -66,10 +66,10 @@ set (H5B2_SRCS
)
set (H5B2_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SOURCES}" )
-set (H5C_SRCS
+set (H5C_SOURCES
${HDF5_SRC_DIR}/H5C.c
)
set (H5C_HDRS
@@ -78,7 +78,7 @@ set (H5C_HDRS
IDE_GENERATED_PROPERTIES ("H5C" "${H5C_HDRS}" "${H5C_SOURCES}" )
-set (H5CS_SRCS
+set (H5CS_SOURCES
${HDF5_SRC_DIR}/H5CS.c
)
set (H5CS_HDRS
@@ -86,7 +86,7 @@ set (H5CS_HDRS
IDE_GENERATED_PROPERTIES ("H5CS" "${H5CS_HDRS}" "${H5CS_SOURCES}" )
-set (H5D_SRCS
+set (H5D_SOURCES
${HDF5_SRC_DIR}/H5D.c
${HDF5_SRC_DIR}/H5Dbtree.c
${HDF5_SRC_DIR}/H5Dchunk.c
@@ -109,9 +109,9 @@ set (H5D_SRCS
set (H5D_HDRS
${HDF5_SRC_DIR}/H5Dpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5D" "${H5D_HDRS}" "${H5D_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5D" "${H5D_HDRS}" "${H5D_SOURCES}" )
-set (H5E_SRCS
+set (H5E_SOURCES
${HDF5_SRC_DIR}/H5E.c
${HDF5_SRC_DIR}/H5Edeprec.c
${HDF5_SRC_DIR}/H5Eint.c
@@ -121,9 +121,9 @@ set (H5E_HDRS
${HDF5_SRC_DIR}/H5Epubgen.h
${HDF5_SRC_DIR}/H5Epublic.h
)
-IDE_GENERATED_PROPERTIES ("H5E" "${H5E_HDRS}" "${H5E_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5E" "${H5E_HDRS}" "${H5E_SOURCES}" )
-set (H5F_SRCS
+set (H5F_SOURCES
${HDF5_SRC_DIR}/H5F.c
${HDF5_SRC_DIR}/H5Faccum.c
${HDF5_SRC_DIR}/H5Fcwfs.c
@@ -144,18 +144,21 @@ set (H5F_SRCS
set (H5F_HDRS
${HDF5_SRC_DIR}/H5Fpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5F" "${H5F_HDRS}" "${H5F_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5F" "${H5F_HDRS}" "${H5F_SOURCES}" )
-set (H5FD_SRCS
+set (H5FD_SOURCES
${HDF5_SRC_DIR}/H5FD.c
${HDF5_SRC_DIR}/H5FDcore.c
${HDF5_SRC_DIR}/H5FDdirect.c
${HDF5_SRC_DIR}/H5FDfamily.c
+ ${HDF5_SRC_DIR}/H5FDhdfs.c
${HDF5_SRC_DIR}/H5FDint.c
${HDF5_SRC_DIR}/H5FDlog.c
${HDF5_SRC_DIR}/H5FDmpi.c
${HDF5_SRC_DIR}/H5FDmpio.c
${HDF5_SRC_DIR}/H5FDmulti.c
+ ${HDF5_SRC_DIR}/H5FDros3.c
+ ${HDF5_SRC_DIR}/H5FDs3comms.c
${HDF5_SRC_DIR}/H5FDsec2.c
${HDF5_SRC_DIR}/H5FDspace.c
${HDF5_SRC_DIR}/H5FDstdio.c
@@ -166,35 +169,38 @@ set (H5FD_HDRS
${HDF5_SRC_DIR}/H5FDcore.h
${HDF5_SRC_DIR}/H5FDdirect.h
${HDF5_SRC_DIR}/H5FDfamily.h
+ ${HDF5_SRC_DIR}/H5FDhdfs.h
${HDF5_SRC_DIR}/H5FDlog.h
${HDF5_SRC_DIR}/H5FDmpi.h
${HDF5_SRC_DIR}/H5FDmpio.h
${HDF5_SRC_DIR}/H5FDmulti.h
${HDF5_SRC_DIR}/H5FDpublic.h
+ ${HDF5_SRC_DIR}/H5FDros3.h
+ ${HDF5_SRC_DIR}/H5FDs3comms.c
${HDF5_SRC_DIR}/H5FDsec2.h
${HDF5_SRC_DIR}/H5FDstdio.h
${HDF5_SRC_DIR}/H5FDwindows.h
)
-IDE_GENERATED_PROPERTIES ("H5FD" "${H5FD_HDRS}" "${H5FD_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FD" "${H5FD_HDRS}" "${H5FD_SOURCES}" )
-set (H5FL_SRCS
+set (H5FL_SOURCES
${HDF5_SRC_DIR}/H5FL.c
)
set (H5FL_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5FL" "${H5FL_HDRS}" "${H5FL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FL" "${H5FL_HDRS}" "${H5FL_SOURCES}" )
-set (H5FO_SRCS
+set (H5FO_SOURCES
${HDF5_SRC_DIR}/H5FO.c
)
set (H5FO_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5FO" "${H5FO_HDRS}" "${H5FO_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FO" "${H5FO_HDRS}" "${H5FO_SOURCES}" )
-set (H5FS_SRCS
+set (H5FS_SOURCES
${HDF5_SRC_DIR}/H5FS.c
${HDF5_SRC_DIR}/H5FScache.c
${HDF5_SRC_DIR}/H5FSdbg.c
@@ -205,9 +211,9 @@ set (H5FS_SRCS
set (H5FS_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5FS" "${H5FS_HDRS}" "${H5FS_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FS" "${H5FS_HDRS}" "${H5FS_SOURCES}" )
-set (H5G_SRCS
+set (H5G_SOURCES
${HDF5_SRC_DIR}/H5G.c
${HDF5_SRC_DIR}/H5Gbtree2.c
${HDF5_SRC_DIR}/H5Gcache.c
@@ -231,9 +237,9 @@ set (H5G_SRCS
set (H5G_HDRS
${HDF5_SRC_DIR}/H5Gpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5G" "${H5G_HDRS}" "${H5G_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5G" "${H5G_HDRS}" "${H5G_SOURCES}" )
-set (H5HF_SRCS
+set (H5HF_SOURCES
${HDF5_SRC_DIR}/H5HF.c
${HDF5_SRC_DIR}/H5HFbtree2.c
${HDF5_SRC_DIR}/H5HFcache.c
@@ -254,9 +260,9 @@ set (H5HF_SRCS
set (H5HF_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HF" "${H5HF_HDRS}" "${H5HF_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HF" "${H5HF_HDRS}" "${H5HF_SOURCES}" )
-set (H5HG_SRCS
+set (H5HG_SOURCES
${HDF5_SRC_DIR}/H5HG.c
${HDF5_SRC_DIR}/H5HGcache.c
${HDF5_SRC_DIR}/H5HGdbg.c
@@ -265,9 +271,9 @@ set (H5HG_SRCS
set (H5HG_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HG" "${H5HG_HDRS}" "${H5HG_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HG" "${H5HG_HDRS}" "${H5HG_SOURCES}" )
-set (H5HL_SRCS
+set (H5HL_SOURCES
${HDF5_SRC_DIR}/H5HL.c
${HDF5_SRC_DIR}/H5HLcache.c
${HDF5_SRC_DIR}/H5HLdbg.c
@@ -276,38 +282,38 @@ set (H5HL_SRCS
set (H5HL_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HL" "${H5HL_HDRS}" "${H5HL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HL" "${H5HL_HDRS}" "${H5HL_SOURCES}" )
-set (H5HP_SRCS
+set (H5HP_SOURCES
${HDF5_SRC_DIR}/H5HP.c
)
set (H5HP_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HP" "${H5HP_HDRS}" "${H5HP_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HP" "${H5HP_HDRS}" "${H5HP_SOURCES}" )
-set (H5I_SRCS
+set (H5I_SOURCES
${HDF5_SRC_DIR}/H5I.c
${HDF5_SRC_DIR}/H5Itest.c
)
set (H5I_HDRS
${HDF5_SRC_DIR}/H5Ipublic.h
)
-IDE_GENERATED_PROPERTIES ("H5I" "${H5I_HDRS}" "${H5I_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5I" "${H5I_HDRS}" "${H5I_SOURCES}" )
-set (H5L_SRCS
+set (H5L_SOURCES
${HDF5_SRC_DIR}/H5L.c
${HDF5_SRC_DIR}/H5Lexternal.c
)
set (H5L_HDRS
${HDF5_SRC_DIR}/H5Lpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5L" "${H5L_HDRS}" "${H5L_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5L" "${H5L_HDRS}" "${H5L_SOURCES}" )
-set (H5MF_SRCS
+set (H5MF_SOURCES
${HDF5_SRC_DIR}/H5MF.c
${HDF5_SRC_DIR}/H5MFaggr.c
${HDF5_SRC_DIR}/H5MFdbg.c
@@ -316,28 +322,28 @@ set (H5MF_SRCS
set (H5MF_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5MF" "${H5MF_HDRS}" "${H5MF_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5MF" "${H5MF_HDRS}" "${H5MF_SOURCES}" )
-set (H5MM_SRCS
+set (H5MM_SOURCES
${HDF5_SRC_DIR}/H5MM.c
)
set (H5MM_HDRS
${HDF5_SRC_DIR}/H5MMpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5MM" "${H5MM_HDRS}" "${H5MM_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5MM" "${H5MM_HDRS}" "${H5MM_SOURCES}" )
-set (H5MP_SRCS
+set (H5MP_SOURCES
${HDF5_SRC_DIR}/H5MP.c
${HDF5_SRC_DIR}/H5MPtest.c
)
set (H5MP_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5MP" "${H5MP_HDRS}" "${H5MP_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5MP" "${H5MP_HDRS}" "${H5MP_SOURCES}" )
-set (H5O_SRCS
+set (H5O_SOURCES
${HDF5_SRC_DIR}/H5O.c
${HDF5_SRC_DIR}/H5Oainfo.c
${HDF5_SRC_DIR}/H5Oalloc.c
@@ -375,9 +381,9 @@ set (H5O_SRCS
set (H5O_HDRS
${HDF5_SRC_DIR}/H5Opublic.h
)
-IDE_GENERATED_PROPERTIES ("H5O" "${H5O_HDRS}" "${H5O_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5O" "${H5O_HDRS}" "${H5O_SOURCES}" )
-set (H5P_SRCS
+set (H5P_SOURCES
${HDF5_SRC_DIR}/H5P.c
${HDF5_SRC_DIR}/H5Pacpl.c
${HDF5_SRC_DIR}/H5Pdapl.c
@@ -400,9 +406,9 @@ set (H5P_SRCS
set (H5P_HDRS
${HDF5_SRC_DIR}/H5Ppublic.h
)
-IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SOURCES}" )
-set (H5PL_SRCS
+set (H5PL_SOURCES
${HDF5_SRC_DIR}/H5PL.c
)
@@ -410,35 +416,36 @@ set (H5PL_HDRS
${HDF5_SRC_DIR}/H5PLextern.h
${HDF5_SRC_DIR}/H5PLpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5PL" "${H5PL_HDRS}" "${H5PL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5PL" "${H5PL_HDRS}" "${H5PL_SOURCES}" )
-set (H5R_SRCS
+set (H5R_SOURCES
${HDF5_SRC_DIR}/H5R.c
${HDF5_SRC_DIR}/H5Rdeprec.c
)
set (H5R_HDRS
${HDF5_SRC_DIR}/H5Rpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5R" "${H5R_HDRS}" "${H5R_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5R" "${H5R_HDRS}" "${H5R_SOURCES}" )
-set (H5RC_SRCS
+set (H5RC_SOURCES
${HDF5_SRC_DIR}/H5RC.c
)
set (H5RC_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5RC" "${H5RC_HDRS}" "${H5RC_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5RC" "${H5RC_HDRS}" "${H5RC_SOURCES}" )
-set (H5RS_SRCS
+set (H5RS_SOURCES
${HDF5_SRC_DIR}/H5RS.c
)
set (H5RS_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5RS" "${H5RS_HDRS}" "${H5RS_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5RS" "${H5RS_HDRS}" "${H5RS_SOURCES}" )
-set (H5S_SRCS
+
+set (H5S_SOURCES
${HDF5_SRC_DIR}/H5S.c
${HDF5_SRC_DIR}/H5Sall.c
${HDF5_SRC_DIR}/H5Sdbg.c
@@ -453,18 +460,18 @@ set (H5S_SRCS
set (H5S_HDRS
${HDF5_SRC_DIR}/H5Spublic.h
)
-IDE_GENERATED_PROPERTIES ("H5S" "${H5S_HDRS}" "${H5S_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5S" "${H5S_HDRS}" "${H5S_SOURCES}" )
-set (H5SL_SRCS
+set (H5SL_SOURCES
${HDF5_SRC_DIR}/H5SL.c
)
set (H5SL_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5SL" "${H5SL_HDRS}" "${H5SL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5SL" "${H5SL_HDRS}" "${H5SL_SOURCES}" )
-set (H5SM_SRCS
+set (H5SM_SOURCES
${HDF5_SRC_DIR}/H5SM.c
${HDF5_SRC_DIR}/H5SMbtree2.c
${HDF5_SRC_DIR}/H5SMcache.c
@@ -474,18 +481,18 @@ set (H5SM_SRCS
set (H5SM_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5SM" "${H5SM_HDRS}" "${H5SM_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5SM" "${H5SM_HDRS}" "${H5SM_SOURCES}" )
-set (H5ST_SRCS
+set (H5ST_SOURCES
${HDF5_SRC_DIR}/H5ST.c
)
set (H5ST_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5ST" "${H5ST_HDRS}" "${H5ST_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5ST" "${H5ST_HDRS}" "${H5ST_SOURCES}" )
-set (H5T_SRCS
+set (H5T_SOURCES
${HDF5_SRC_DIR}/H5T.c
${HDF5_SRC_DIR}/H5Tarray.c
${HDF5_SRC_DIR}/H5Tbit.c
@@ -514,34 +521,34 @@ set (H5T_SRCS
set (H5T_HDRS
${HDF5_SRC_DIR}/H5Tpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5T" "${H5T_HDRS}" "${H5T_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5T" "${H5T_HDRS}" "${H5T_SOURCES}" )
-set (H5TS_SRCS
+set (H5TS_SOURCES
${HDF5_SRC_DIR}/H5TS.c
)
set (H5TS_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5TS" "${H5TS_HDRS}" "${H5TS_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5TS" "${H5TS_HDRS}" "${H5TS_SOURCES}" )
-set (H5VM_SRCS
+set (H5VM_SOURCES
${HDF5_SRC_DIR}/H5VM.c
)
set (H5VM_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5VM" "${H5VM_HDRS}" "${H5VM_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5VM" "${H5VM_HDRS}" "${H5VM_SOURCES}" )
-set (H5WB_SRCS
+set (H5WB_SOURCES
${HDF5_SRC_DIR}/H5WB.c
)
set (H5WB_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5WB" "${H5WB_HDRS}" "${H5WB_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5WB" "${H5WB_HDRS}" "${H5WB_SOURCES}" )
-set (H5Z_SRCS
+set (H5Z_SOURCES
${HDF5_SRC_DIR}/H5Z.c
${HDF5_SRC_DIR}/H5Zdeflate.c
${HDF5_SRC_DIR}/H5Zfletcher32.c
@@ -560,48 +567,48 @@ endif ()
set (H5Z_HDRS
${HDF5_SRC_DIR}/H5Zpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5Z" "${H5Z_HDRS}" "${H5Z_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5Z" "${H5Z_HDRS}" "${H5Z_SOURCES}" )
set (common_SRCS
- ${H5_SRCS}
- ${H5A_SRCS}
- ${H5AC_SRCS}
- ${H5B_SRCS}
- ${H5B2_SRCS}
- ${H5C_SRCS}
- ${H5CS_SRCS}
- ${H5D_SRCS}
- ${H5E_SRCS}
- ${H5F_SRCS}
- ${H5FD_SRCS}
- ${H5FL_SRCS}
- ${H5FO_SRCS}
- ${H5FS_SRCS}
- ${H5G_SRCS}
- ${H5HF_SRCS}
- ${H5HG_SRCS}
- ${H5HL_SRCS}
- ${H5HP_SRCS}
- ${H5I_SRCS}
- ${H5L_SRCS}
- ${H5MF_SRCS}
- ${H5MM_SRCS}
- ${H5MP_SRCS}
- ${H5O_SRCS}
- ${H5P_SRCS}
- ${H5PL_SRCS}
- ${H5R_SRCS}
- ${H5RC_SRCS}
- ${H5RS_SRCS}
- ${H5S_SRCS}
- ${H5SL_SRCS}
- ${H5SM_SRCS}
- ${H5ST_SRCS}
- ${H5T_SRCS}
- ${H5TS_SRCS}
- ${H5VM_SRCS}
- ${H5WB_SRCS}
- ${H5Z_SRCS}
+ ${H5_SOURCES}
+ ${H5A_SOURCES}
+ ${H5AC_SOURCES}
+ ${H5B_SOURCES}
+ ${H5B2_SOURCES}
+ ${H5C_SOURCES}
+ ${H5CS_SOURCES}
+ ${H5D_SOURCES}
+ ${H5E_SOURCES}
+ ${H5F_SOURCES}
+ ${H5FD_SOURCES}
+ ${H5FL_SOURCES}
+ ${H5FO_SOURCES}
+ ${H5FS_SOURCES}
+ ${H5G_SOURCES}
+ ${H5HF_SOURCES}
+ ${H5HG_SOURCES}
+ ${H5HL_SOURCES}
+ ${H5HP_SOURCES}
+ ${H5I_SOURCES}
+ ${H5L_SOURCES}
+ ${H5MF_SOURCES}
+ ${H5MM_SOURCES}
+ ${H5MP_SOURCES}
+ ${H5O_SOURCES}
+ ${H5P_SOURCES}
+ ${H5PL_SOURCES}
+ ${H5R_SOURCES}
+ ${H5RC_SOURCES}
+ ${H5RS_SOURCES}
+ ${H5S_SOURCES}
+ ${H5SL_SOURCES}
+ ${H5SM_SOURCES}
+ ${H5ST_SOURCES}
+ ${H5T_SOURCES}
+ ${H5TS_SOURCES}
+ ${H5VM_SOURCES}
+ ${H5WB_SOURCES}
+ ${H5Z_SOURCES}
)
set (H5_PUBLIC_HEADERS
@@ -797,6 +804,27 @@ if (BUILD_SHARED_LIBS)
file (MAKE_DIRECTORY "${HDF5_BINARY_DIR}/shared")
endif ()
+if (LOCAL_BATCH_TEST)
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
+ configure_file (
+ ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_COMMAND}
+ ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_COMMAND} ESCAPE_QUOTES @ONLY
+ )
+ endif ()
+ if (LOCAL_BATCH_SCRIPT_NAME)
+ configure_file (
+ ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_NAME}.in.cmake
+ ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_NAME} ESCAPE_QUOTES @ONLY
+ )
+ endif ()
+ if (LOCAL_BATCH_SCRIPT_PARALLEL_NAME)
+ configure_file (
+ ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}.in.cmake
+ ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME} ESCAPE_QUOTES @ONLY
+ )
+ endif ()
+endif ()
+
if (NOT EXISTS "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
add_executable (H5detect ${HDF5_SRC_DIR}/H5detect.c)
target_include_directories (H5detect PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
@@ -941,7 +969,6 @@ if (NOT ONLY_SHARED_LIBS)
PUBLIC
${HDF_EXTRA_C_FLAGS}
${HDF_EXTRA_FLAGS}
- $<IF:$<CONFIG:Debug>,DEBUG,NDEBUG>
PRIVATE
$<$<BOOL:${HDF5_ENABLE_TRACE}>:H5_DEBUG_API> # Enable tracing of the API
$<$<BOOL:${HDF5_ENABLE_DEBUG_APIS}>:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG>
@@ -971,6 +998,7 @@ if (BUILD_SHARED_LIBS)
add_library (${HDF5_LIBSH_TARGET} SHARED ${common_SRCS} ${shared_gen_SRCS} ${H5_PUBLIC_HEADERS} ${H5_PRIVATE_HEADERS} ${H5_GENERATED_HEADERS})
target_include_directories (${HDF5_LIBSH_TARGET}
PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>"
+ PUBLIC "$<$<BOOL:${HDF5_ENABLE_HDFS}>:${HDFS_INCLUDE_DIR}>"
INTERFACE "$<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>"
)
target_compile_definitions(${HDF5_LIBSH_TARGET}
@@ -978,7 +1006,6 @@ if (BUILD_SHARED_LIBS)
"H5_BUILT_AS_DYNAMIC_LIB"
${HDF_EXTRA_C_FLAGS}
${HDF_EXTRA_FLAGS}
- $<IF:$<CONFIG:Debug>,DEBUG,NDEBUG>
PRIVATE
$<$<BOOL:${HDF5_ENABLE_THREADSAFE}>:H5_HAVE_THREADSAFE>
$<$<BOOL:${HDF5_ENABLE_TRACE}>:H5_DEBUG_API> # Enable tracing of the API
diff --git a/src/H5FD.c b/src/H5FD.c
index 63b7dfc..11ef24a 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -15,7 +15,7 @@
* Programmer: Robb Matzke <matzke@llnl.gov>
* Monday, July 26, 1999
*
- * Purpose: The Virtual File Layer as described in documentation.
+ * Purpose: The Virtual File Layer as described in documentation.
* This is the greatest common denominator for all types of
* storage access whether a file, memory, network, etc. This
* layer usually just dispatches the request to an actual
@@ -26,35 +26,35 @@
/* Module Setup */
/****************/
-#define H5F_PACKAGE /*suppress error about including H5Fpkg */
-#define H5FD_PACKAGE /*suppress error about including H5FDpkg */
+#define H5F_PACKAGE /*suppress error about including H5Fpkg */
+#define H5FD_PACKAGE /*suppress error about including H5FDpkg */
/* Interface initialization */
-#define H5_INTERFACE_INIT_FUNC H5FD_init_interface
+#define H5_INTERFACE_INIT_FUNC H5FD_init_interface
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
-#include "H5Dprivate.h" /* Datasets */
-#include "H5Eprivate.h" /* Error handling */
-#include "H5Fpkg.h" /* File access */
-#include "H5FDpkg.h" /* File Drivers */
-#include "H5FDcore.h" /* Files stored entirely in memory */
-#include "H5FDfamily.h" /* File families */
-#include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */
-#include "H5FDmpi.h" /* MPI-based file drivers */
-#include "H5FDmulti.h" /* Usage-partitioned file family */
-#include "H5FDsec2.h" /* POSIX unbuffered file I/O */
-#include "H5FDstdio.h" /* Standard C buffered I/O */
+#include "H5private.h" /* Generic Functions */
+#include "H5Dprivate.h" /* Datasets */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* File access */
+#include "H5FDpkg.h" /* File Drivers */
+#include "H5FDcore.h" /* Files stored entirely in memory */
+#include "H5FDfamily.h" /* File families */
+#include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */
+#include "H5FDmpi.h" /* MPI-based file drivers */
+#include "H5FDmulti.h" /* Usage-partitioned file family */
+#include "H5FDsec2.h" /* POSIX unbuffered file I/O */
+#include "H5FDstdio.h" /* Standard C buffered I/O */
#ifdef H5_HAVE_WINDOWS
#include "H5FDwindows.h" /* Windows buffered I/O */
#endif
-#include "H5FDdirect.h" /* Direct file I/O */
-#include "H5Iprivate.h" /* IDs */
-#include "H5MMprivate.h" /* Memory management */
-#include "H5Pprivate.h" /* Property lists */
+#include "H5FDdirect.h" /* Direct file I/O */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
/****************/
/* Local Macros */
@@ -113,23 +113,23 @@ static unsigned long H5FD_file_serial_no_g;
/* File driver ID class */
static const H5I_class_t H5I_VFL_CLS[1] = {{
- H5I_VFL, /* ID class value */
- H5I_CLASS_REUSE_IDS, /* Class flags */
- 0, /* # of reserved IDs for class */
- (H5I_free_t)H5FD_free_cls /* Callback routine for closing objects of this class */
+ H5I_VFL, /* ID class value */
+ H5I_CLASS_REUSE_IDS, /* Class flags */
+ 0, /* # of reserved IDs for class */
+ (H5I_free_t)H5FD_free_cls /* Callback routine for closing objects of this class */
}};
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_init
+ * Function: H5FD_init
*
- * Purpose: Initialize the interface from some other package.
+ * Purpose: Initialize the interface from some other package.
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 3, 2007
*
*-------------------------------------------------------------------------
@@ -146,17 +146,17 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_init() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_init_interface
+ * Function: H5FD_init_interface
*
- * Purpose: Initialize the virtual file layer.
+ * Purpose: Initialize the virtual file layer.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -171,7 +171,7 @@ H5FD_init_interface(void)
FUNC_ENTER_NOAPI_NOINIT
if(H5I_register_type(H5I_VFL_CLS) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize interface")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize interface")
/* Reset the file serial numbers */
H5FD_file_serial_no_g = 0;
@@ -180,21 +180,21 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_init_interface() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_term_interface
+ * Function: H5FD_term_interface
*
- * Purpose: Terminate this interface: free all memory and reset global
- * variables to their initial values. Release all ID groups
- * associated with this interface.
+ * Purpose: Terminate this interface: free all memory and reset global
+ * variables to their initial values. Release all ID groups
+ * associated with this interface.
*
- * Return: Success: Positive if anything was done that might
- * have affected other interfaces; zero
- * otherwise.
+ * Return: Success: Positive if anything was done that might
+ * have affected other interfaces; zero
+ * otherwise.
*
- * Failure: Never fails.
+ * Failure: Never fails.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, February 19, 1999
*
* Modifications:
@@ -204,13 +204,13 @@ done:
int
H5FD_term_interface(void)
{
- int n = 0;
+ int n = 0;
FUNC_ENTER_NOAPI_NOINIT_NOERR
if(H5_interface_initialize_g) {
- if(H5I_nmembers(H5I_VFL) > 0) {
- (void)H5I_clear_type(H5I_VFL, FALSE, FALSE);
+ if(H5I_nmembers(H5I_VFL) > 0) {
+ (void)H5I_clear_type(H5I_VFL, FALSE, FALSE);
/* Reset the VFL drivers, if they've been closed */
if(H5I_nmembers(H5I_VFL)==0) {
@@ -232,33 +232,33 @@ H5FD_term_interface(void)
} /* end if */
n++; /*H5I*/
- } /* end if */
+ } /* end if */
else {
/* Destroy the VFL driver id group */
- (void)H5I_dec_type_ref(H5I_VFL);
+ (void)H5I_dec_type_ref(H5I_VFL);
n++; /*H5I*/
- /* Mark closed */
- H5_interface_initialize_g = 0;
- } /* end else */
+ /* Mark closed */
+ H5_interface_initialize_g = 0;
+ } /* end else */
} /* end if */
FUNC_LEAVE_NOAPI(n)
} /* end H5FD_term_interface() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_free_cls
+ * Function: H5FD_free_cls
*
- * Purpose: Frees a file driver class struct and returns an indication of
- * success. This function is used as the free callback for the
- * virtual file layer object identifiers (cf H5FD_init_interface).
+ * Purpose: Frees a file driver class struct and returns an indication of
+ * success. This function is used as the free callback for the
+ * virtual file layer object identifiers (cf H5FD_init_interface).
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -275,22 +275,22 @@ H5FD_free_cls(H5FD_class_t *cls)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_free_cls() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDregister
+ * Function: H5FDregister
*
- * Purpose: Registers a new file driver as a member of the virtual file
- * driver class. Certain fields of the class struct are
- * required and that is checked here so it doesn't have to be
- * checked every time the field is accessed.
+ * Purpose: Registers a new file driver as a member of the virtual file
+ * driver class. Certain fields of the class struct are
+ * required and that is checked here so it doesn't have to be
+ * checked every time the field is accessed.
*
- * Return: Success: A file driver ID which is good until the
- * library is closed or the driver is
- * unregistered.
+ * Return: Success: A file driver ID which is good until the
+ * library is closed or the driver is
+ * unregistered.
*
- * Failure: A negative value.
+ * Failure: A negative value.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -303,26 +303,26 @@ H5FD_free_cls(H5FD_class_t *cls)
hid_t
H5FDregister(const H5FD_class_t *cls)
{
- hid_t ret_value;
- H5FD_mem_t type;
+ hid_t ret_value;
+ H5FD_mem_t type;
FUNC_ENTER_API(FAIL)
H5TRACE1("i", "*x", cls);
/* Check arguments */
if(!cls)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "null class pointer is disallowed")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "null class pointer is disallowed")
if(!cls->open || !cls->close)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`open' and/or `close' methods are not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`open' and/or `close' methods are not defined")
if(!cls->get_eoa || !cls->set_eoa)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eoa' and/or `set_eoa' methods are not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eoa' and/or `set_eoa' methods are not defined")
if(!cls->get_eof)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eof' method is not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eof' method is not defined")
if(!cls->read || !cls->write)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`read' and/or `write' method is not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`read' and/or `write' method is not defined")
for (type=H5FD_MEM_DEFAULT; type<H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t,type))
- if(cls->fl_map[type]<H5FD_MEM_NOLIST || cls->fl_map[type]>=H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid free-list mapping")
+ if(cls->fl_map[type]<H5FD_MEM_NOLIST || cls->fl_map[type]>=H5FD_MEM_NTYPES)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid free-list mapping")
/* Create the new class ID */
if((ret_value=H5FD_register(cls, sizeof(H5FD_class_t), TRUE)) < 0)
@@ -332,22 +332,22 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDregister() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_register
+ * Function: H5FD_register
*
- * Purpose: Registers a new file driver as a member of the virtual file
- * driver class. Certain fields of the class struct are
- * required and that is checked here so it doesn't have to be
- * checked every time the field is accessed.
+ * Purpose: Registers a new file driver as a member of the virtual file
+ * driver class. Certain fields of the class struct are
+ * required and that is checked here so it doesn't have to be
+ * checked every time the field is accessed.
*
- * Return: Success: A file driver ID which is good until the
- * library is closed or the driver is
- * unregistered.
+ * Return: Success: A file driver ID which is good until the
+ * library is closed or the driver is
+ * unregistered.
*
- * Failure: A negative value.
+ * Failure: A negative value.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -363,10 +363,10 @@ done:
hid_t
H5FD_register(const void *_cls, size_t size, hbool_t app_ref)
{
- const H5FD_class_t *cls = (const H5FD_class_t *)_cls;
- H5FD_class_t *saved = NULL;
- H5FD_mem_t type;
- hid_t ret_value;
+ const H5FD_class_t *cls = (const H5FD_class_t *)_cls;
+ H5FD_class_t *saved = NULL;
+ H5FD_mem_t type;
+ hid_t ret_value;
FUNC_ENTER_NOAPI(FAIL)
@@ -381,7 +381,7 @@ H5FD_register(const void *_cls, size_t size, hbool_t app_ref)
/* Copy the class structure so the caller can reuse or free it */
if(NULL == (saved = (H5FD_class_t *)H5MM_malloc(size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for file driver class struct")
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for file driver class struct")
HDmemcpy(saved, cls, size);
/* Create the new class ID */
@@ -396,20 +396,20 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_register() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDunregister
+ * Function: H5FDunregister
*
- * Purpose: Removes a driver ID from the library. This in no way affects
- * file access property lists which have been defined to use
- * this driver or files which are already opened under this
- * driver.
+ * Purpose: Removes a driver ID from the library. This in no way affects
+ * file access property lists which have been defined to use
+ * this driver or files which are already opened under this
+ * driver.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
*-------------------------------------------------------------------------
@@ -424,33 +424,33 @@ H5FDunregister(hid_t driver_id)
/* Check arguments */
if(NULL == H5I_object_verify(driver_id, H5I_VFL))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver")
/* The H5FD_class_t struct will be freed by this function */
if(H5I_dec_app_ref(driver_id) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "unable to unregister file driver")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "unable to unregister file driver")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDunregister() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_class
+ * Function: H5FD_get_class
*
- * Purpose: Obtains a pointer to the driver struct containing all the
- * callback pointers, etc. The PLIST_ID argument can be a file
- * access property list, a data transfer property list, or a
- * file driver identifier.
+ * Purpose: Obtains a pointer to the driver struct containing all the
+ * callback pointers, etc. The PLIST_ID argument can be a file
+ * access property list, a data transfer property list, or a
+ * file driver identifier.
*
- * Return: Success: Ptr to the driver information. The pointer is
- * only valid as long as the driver remains
- * registered or some file or property list
- * exists which references the driver.
+ * Return: Success: Ptr to the driver information. The pointer is
+ * only valid as long as the driver remains
+ * registered or some file or property list
+ * exists which references the driver.
*
- * Failure: NULL
+ * Failure: NULL
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, August 20, 1999
*
*-------------------------------------------------------------------------
@@ -458,12 +458,12 @@ done:
H5FD_class_t *
H5FD_get_class(hid_t id)
{
- H5FD_class_t *ret_value = NULL;
+ H5FD_class_t *ret_value = NULL;
FUNC_ENTER_NOAPI(NULL)
if(H5I_VFL == H5I_get_type(id))
- ret_value = (H5FD_class_t *)H5I_object(id);
+ ret_value = (H5FD_class_t *)H5I_object(id);
else {
H5P_genplist_t *plist; /* Property list pointer */
hid_t driver_id = -1;
@@ -485,19 +485,19 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_class() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_sb_size
+ * Function: H5FD_sb_size
*
- * Purpose: Obtains the number of bytes required to store the driver file
- * access data in the HDF5 superblock.
+ * Purpose: Obtains the number of bytes required to store the driver file
+ * access data in the HDF5 superblock.
*
- * Return: Success: Number of bytes required.
+ * Return: Success: Number of bytes required.
*
- * Failure: 0 if an error occurs or if the driver has no
- * data to store in the superblock.
+ * Failure: 0 if an error occurs or if the driver has no
+ * data to store in the superblock.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 16, 1999
*
* Modifications:
@@ -507,35 +507,35 @@ done:
hsize_t
H5FD_sb_size(H5FD_t *file)
{
- hsize_t ret_value=0;
+ hsize_t ret_value=0;
FUNC_ENTER_NOAPI(0)
HDassert(file && file->cls);
if(file->cls->sb_size)
- ret_value = (file->cls->sb_size)(file);
+ ret_value = (file->cls->sb_size)(file);
done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_sb_encode
+ * Function: H5FD_sb_encode
*
- * Purpose: Encode driver-specific data into the output arguments. The
- * NAME is a nine-byte buffer which should get an
- * eight-character driver name and/or version followed by a null
- * terminator. The BUF argument is a buffer to receive the
- * encoded driver-specific data. The size of the BUF array is
- * the size returned by the H5FD_sb_size() call.
+ * Purpose: Encode driver-specific data into the output arguments. The
+ * NAME is a nine-byte buffer which should get an
+ * eight-character driver name and/or version followed by a null
+ * terminator. The BUF argument is a buffer to receive the
+ * encoded driver-specific data. The size of the BUF array is
+ * the size returned by the H5FD_sb_size() call.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 16, 1999
*
* Modifications:
@@ -552,22 +552,22 @@ H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf)
HDassert(file && file->cls);
if(file->cls->sb_encode &&
(file->cls->sb_encode)(file, name/*out*/, buf/*out*/) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_sb_decode
+ * Function: H5FD_sb_decode
*
- * Purpose: Decodes the driver information block.
+ * Purpose: Decodes the driver information block.
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 16, 1999
*
*-------------------------------------------------------------------------
@@ -581,25 +581,25 @@ H5FD_sb_decode(H5FD_t *file, const char *name, const uint8_t *buf)
HDassert(file && file->cls);
if(file->cls->sb_decode && (file->cls->sb_decode)(file, name, buf) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_sb_decode() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_pl_copy
+ * Function: H5FD_pl_copy
*
- * Purpose: Copies the driver-specific part of the a property list.
+ * Purpose: Copies the driver-specific part of the a property list.
* This is common code, used by both the dataset transfer and
* file access property list routines.
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, October 23, 2003
*
* Modifications:
@@ -638,18 +638,18 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_pl_copy() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_pl_close
+ * Function: H5FD_pl_close
*
- * Purpose: Closes a driver for a property list
+ * Purpose: Closes a driver for a property list
* This is common code, used by both the dataset transfer and
* file access property list routines.
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, October 23, 2003
*
*-------------------------------------------------------------------------
@@ -663,11 +663,11 @@ H5FD_pl_close(hid_t driver_id, herr_t (*free_func)(void *), void *pl)
/* Allow driver to free or do it ourselves */
if(pl && free_func) {
- if((free_func)(pl) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver free request failed")
+ if((free_func)(pl) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver free request failed")
} /* end if */
else
- H5MM_xfree(pl);
+ H5MM_xfree(pl);
/* Decrement reference count for driver */
if(H5I_dec_ref(driver_id) < 0)
@@ -677,27 +677,27 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_pl_close() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_get
+ * Function: H5FD_fapl_get
*
- * Purpose: Gets the file access property list associated with a file.
- * Usually the file will copy what it needs from the original
- * file access property list when the file is created. The
- * purpose of this function is to create a new file access
- * property list based on the settings in the file, which may
- * have been modified from the original file access property
- * list.
+ * Purpose: Gets the file access property list associated with a file.
+ * Usually the file will copy what it needs from the original
+ * file access property list when the file is created. The
+ * purpose of this function is to create a new file access
+ * property list based on the settings in the file, which may
+ * have been modified from the original file access property
+ * list.
*
- * Return: Success: Pointer to a new file access property list
- * with all members copied. If the file is
- * closed then this property list lives on, and
- * vice versa.
+ * Return: Success: Pointer to a new file access property list
+ * with all members copied. If the file is
+ * closed then this property list lives on, and
+ * vice versa.
*
- * Failure: NULL, including when the file has no
- * properties.
+ * Failure: NULL, including when the file has no
+ * properties.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, August 13, 1999
*
* Modifications:
@@ -707,30 +707,30 @@ done:
void *
H5FD_fapl_get(H5FD_t *file)
{
- void *ret_value=NULL;
+ void *ret_value=NULL;
FUNC_ENTER_NOAPI(NULL)
HDassert(file);
if(file->cls->fapl_get)
- ret_value = (file->cls->fapl_get)(file);
+ ret_value = (file->cls->fapl_get)(file);
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_fapl_get() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_open
+ * Function: H5FD_fapl_open
*
- * Purpose: Mark a driver as used by a file access property list
+ * Purpose: Mark a driver as used by a file access property list
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, October 23, 2003
*
* Modifications:
@@ -766,18 +766,18 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_fapl_open() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_copy
+ * Function: H5FD_fapl_copy
*
- * Purpose: Copies the driver-specific part of the file access property
- * list.
+ * Purpose: Copies the driver-specific part of the file access property
+ * list.
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, August 3, 1999
*
*-------------------------------------------------------------------------
@@ -792,7 +792,7 @@ H5FD_fapl_copy(hid_t driver_id, const void *old_fapl, void **copied_fapl)
/* Check args */
if(NULL == (driver = (H5FD_class_t *)H5I_object(driver_id)))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a driver ID")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a driver ID")
/* Copy the file access property list */
if(H5FD_pl_copy(driver->fapl_copy, driver->fapl_size, old_fapl, copied_fapl) < 0)
@@ -802,16 +802,16 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_close
+ * Function: H5FD_fapl_close
*
- * Purpose: Closes a driver for a dataset transfer property list
+ * Purpose: Closes a driver for a dataset transfer property list
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, August 3, 1999
*
* Modifications:
@@ -821,7 +821,7 @@ done:
herr_t
H5FD_fapl_close(hid_t driver_id, void *fapl)
{
- H5FD_class_t *driver = NULL;
+ H5FD_class_t *driver = NULL;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
@@ -840,56 +840,56 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_fapl_close() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDopen
- *
- * Purpose: Opens a file named NAME for the type(s) of access described
- * by the bit vector FLAGS according to a file access property
- * list FAPL_ID (which may be the constant H5P_DEFAULT). The
- * file should expect to handle format addresses in the range [0,
- * MAXADDR] (if MAXADDR is the undefined address then the caller
- * doesn't care about the address range).
- *
- * Possible values for the FLAGS bits are:
- *
- * H5F_ACC_RDWR: Open the file for read and write access. If
- * this bit is not set then open the file for
- * read only access. It is permissible to open a
- * file for read and write access when only read
- * access is requested by the library (the
- * library will never attempt to write to a file
- * which it opened with only read access).
- *
- * H5F_ACC_CREATE: Create the file if it doesn't already exist.
- * However, see H5F_ACC_EXCL below.
- *
- * H5F_ACC_TRUNC: Truncate the file if it already exists. This
- * is equivalent to deleting the file and then
- * creating a new empty file.
- *
- * H5F_ACC_EXCL: When used with H5F_ACC_CREATE, if the file
- * already exists then the open should fail.
- * Note that this is unsupported/broken with
- * some file drivers (e.g., sec2 across nfs) and
- * will contain a race condition when used to
- * perform file locking.
- *
- * The MAXADDR is the maximum address which will be requested by
- * the library during an allocation operation. Usually this is
- * the same value as the MAXADDR field of the class structure,
- * but it can be smaller if the driver is being used under some
- * other driver.
- *
- * Note that when the driver `open' callback gets control that
- * the public part of the file struct (the H5FD_t part) will be
- * incomplete and will be filled in after that callback returns.
- *
- * Return: Success: Pointer to a new file driver struct.
- *
- * Failure: NULL
- *
- * Programmer: Robb Matzke
+ * Function: H5FDopen
+ *
+ * Purpose: Opens a file named NAME for the type(s) of access described
+ * by the bit vector FLAGS according to a file access property
+ * list FAPL_ID (which may be the constant H5P_DEFAULT). The
+ * file should expect to handle format addresses in the range [0,
+ * MAXADDR] (if MAXADDR is the undefined address then the caller
+ * doesn't care about the address range).
+ *
+ * Possible values for the FLAGS bits are:
+ *
+ * H5F_ACC_RDWR: Open the file for read and write access. If
+ * this bit is not set then open the file for
+ * read only access. It is permissible to open a
+ * file for read and write access when only read
+ * access is requested by the library (the
+ * library will never attempt to write to a file
+ * which it opened with only read access).
+ *
+ * H5F_ACC_CREATE: Create the file if it doesn't already exist.
+ * However, see H5F_ACC_EXCL below.
+ *
+ * H5F_ACC_TRUNC: Truncate the file if it already exists. This
+ * is equivalent to deleting the file and then
+ * creating a new empty file.
+ *
+ * H5F_ACC_EXCL: When used with H5F_ACC_CREATE, if the file
+ * already exists then the open should fail.
+ * Note that this is unsupported/broken with
+ * some file drivers (e.g., sec2 across nfs) and
+ * will contain a race condition when used to
+ * perform file locking.
+ *
+ * The MAXADDR is the maximum address which will be requested by
+ * the library during an allocation operation. Usually this is
+ * the same value as the MAXADDR field of the class structure,
+ * but it can be smaller if the driver is being used under some
+ * other driver.
+ *
+ * Note that when the driver `open' callback gets control that
+ * the public part of the file struct (the H5FD_t part) will be
+ * incomplete and will be filled in after that callback returns.
+ *
+ * Return: Success: Pointer to a new file driver struct.
+ *
+ * Failure: NULL
+ *
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
* Modifications:
@@ -899,7 +899,7 @@ done:
H5FD_t *
H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
- H5FD_t *ret_value=NULL;
+ H5FD_t *ret_value=NULL;
FUNC_ENTER_API(NULL)
H5TRACE4("*x", "*sIuia", name, flags, fapl_id, maxaddr);
@@ -912,50 +912,50 @@ H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
if(NULL==(ret_value=H5FD_open(name, flags, fapl_id, maxaddr)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "unable to open file")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "unable to open file")
done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_open
+ * Function: H5FD_open
*
- * Purpose: Private version of H5FDopen()
+ * Purpose: Private version of H5FDopen()
*
- * Return: Success: Pointer to a new file driver struct
+ * Return: Success: Pointer to a new file driver struct
*
- * Failure: NULL
+ * Failure: NULL
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
* Modifications:
*
- * Raymond Lu
- * Tuesday, Oct 23, 2001
- * Changed the file access list to the new generic property
- * list.
+ * Raymond Lu
+ * Tuesday, Oct 23, 2001
+ * Changed the file access list to the new generic property
+ * list.
*
*-------------------------------------------------------------------------
*/
H5FD_t *
H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
- H5FD_class_t *driver; /* VFD for file */
- H5FD_t *file = NULL; /* VFD file struct */
+ H5FD_class_t *driver; /* VFD for file */
+ H5FD_t *file = NULL; /* VFD file struct */
hid_t driver_id = -1; /* VFD ID */
H5P_genplist_t *plist; /* Property list pointer */
unsigned long driver_flags = 0; /* File-inspecific driver feature flags */
H5FD_file_image_info_t file_image_info; /* Initial file image */
- H5FD_t *ret_value; /* Return value */
+ H5FD_t *ret_value; /* Return value */
FUNC_ENTER_NOAPI(NULL)
/* Sanity check */
if(0 == maxaddr)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "zero format address range")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "zero format address range")
/* Get file access property list */
if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
@@ -967,9 +967,9 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
/* Get driver info */
if(NULL == (driver = (H5FD_class_t *)H5I_object(driver_id)))
- HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid driver ID in file access property list")
+ HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid driver ID in file access property list")
if(NULL == driver->open)
- HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "file driver has no `open' method")
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "file driver has no `open' method")
/* Query driver flag */
H5FD_driver_query(driver, &driver_flags);
@@ -988,7 +988,7 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
if(HADDR_UNDEF == maxaddr)
maxaddr = driver->maxaddr;
if(NULL == (file = (driver->open)(name, flags, fapl_id, maxaddr)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed")
/*
* Fill in public fields. We must increment the reference count on the
@@ -1027,20 +1027,20 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_open() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDclose
+ * Function: H5FDclose
*
* Purpose: Closes the file by calling the driver `close' callback, which
- * should free all driver-private data and free the file struct.
- * Note that the public part of the file struct (the H5FD_t part)
- * will be all zero during the driver close callback like during
- * the `open' callback.
+ * should free all driver-private data and free the file struct.
+ * Note that the public part of the file struct (the H5FD_t part)
+ * will be all zero during the driver close callback like during
+ * the `open' callback.
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
*-------------------------------------------------------------------------
@@ -1054,25 +1054,25 @@ H5FDclose(H5FD_t *file)
H5TRACE1("e", "*x", file);
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(H5FD_close(file) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDclose() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_close
+ * Function: H5FD_close
*
- * Purpose: Private version of H5FDclose()
+ * Purpose: Private version of H5FDclose()
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
*-------------------------------------------------------------------------
@@ -1105,25 +1105,25 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_close() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDcmp
+ * Function: H5FDcmp
*
- * Purpose: Compare the keys of two files using the file driver callback
- * if the files belong to the same driver, otherwise sort the
- * files by driver class pointer value.
+ * Purpose: Compare the keys of two files using the file driver callback
+ * if the files belong to the same driver, otherwise sort the
+ * files by driver class pointer value.
*
- * Return: Success: A value like strcmp()
+ * Return: Success: A value like strcmp()
*
- * Failure: Must never fail. If both file handles are
- * invalid then they compare equal. If one file
- * handle is invalid then it compares less than
- * the other. If both files belong to the same
- * driver and the driver doesn't provide a
- * comparison callback then the file pointers
- * themselves are compared.
+ * Failure: Must never fail. If both file handles are
+ * invalid then they compare equal. If one file
+ * handle is invalid then it compares less than
+ * the other. If both files belong to the same
+ * driver and the driver doesn't provide a
+ * comparison callback then the file pointers
+ * themselves are compared.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
* Modifications:
@@ -1133,7 +1133,7 @@ done:
int
H5FDcmp(const H5FD_t *f1, const H5FD_t *f2)
{
- int ret_value;
+ int ret_value;
FUNC_ENTER_API(-1) /*return value is arbitrary*/
H5TRACE2("Is", "*x*x", f1, f2);
@@ -1144,17 +1144,17 @@ done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_cmp
+ * Function: H5FD_cmp
*
- * Purpose: Private version of H5FDcmp()
+ * Purpose: Private version of H5FDcmp()
*
- * Return: Success: A value like strcmp()
+ * Return: Success: A value like strcmp()
*
- * Failure: Must never fail.
+ * Failure: Must never fail.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
* Modifications:
@@ -1164,7 +1164,7 @@ done:
int
H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2)
{
- int ret_value;
+ int ret_value;
FUNC_ENTER_NOAPI(-1) /*return value is arbitrary*/
@@ -1181,11 +1181,11 @@ H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2)
/* Files are same driver; no cmp callback */
if(!f1->cls->cmp) {
- if(f1<f2)
+ if(f1<f2)
HGOTO_DONE(-1)
- if(f1>f2)
+ if(f1>f2)
HGOTO_DONE(1)
- HGOTO_DONE(0)
+ HGOTO_DONE(0)
}
ret_value = (f1->cls->cmp)(f1, f2);
@@ -1194,17 +1194,17 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDquery
+ * Function: H5FDquery
*
- * Purpose: Query a VFL driver for its feature flags. (listed in H5FDpublic.h)
+ * Purpose: Query a VFL driver for its feature flags. (listed in H5FDpublic.h)
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Friday, August 25, 2000
*
* Modifications:
@@ -1214,7 +1214,7 @@ done:
int
H5FDquery(const H5FD_t *f, unsigned long *flags/*out*/)
{
- int ret_value;
+ int ret_value;
FUNC_ENTER_API(FAIL)
H5TRACE2("Is", "*xx", f, flags);
@@ -1228,17 +1228,17 @@ done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_query
+ * Function: H5FD_query
*
- * Purpose: Private version of H5FDquery()
+ * Purpose: Private version of H5FDquery()
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Friday, August 25, 2000
*
*-------------------------------------------------------------------------
@@ -1246,7 +1246,7 @@ done:
static int
H5FD_query(const H5FD_t *f, unsigned long *flags/*out*/)
{
- int ret_value = 0; /* Return value */
+ int ret_value = 0; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOERR
@@ -1262,7 +1262,7 @@ H5FD_query(const H5FD_t *f, unsigned long *flags/*out*/)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_query() */
-
+
/*-------------------------------------------------------------------------
* Function: H5FD_driver_query
*
@@ -1292,48 +1292,48 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags/*out*/)
/* Check for the driver to query and then query it */
if(driver->query)
ret_value = (driver->query)(NULL, flags);
- else
+ else
*flags = 0;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_driver_query() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDalloc
+ * Function: H5FDalloc
*
- * Purpose: Allocates SIZE bytes of memory from the FILE. The memory will
- * be used according to the allocation class TYPE. First we try
- * to satisfy the request from one of the free lists, according
- * to the free list map provided by the driver. The free list
- * array has one entry for each request type and the value of
- * that array element can be one of four possibilities:
+ * Purpose: Allocates SIZE bytes of memory from the FILE. The memory will
+ * be used according to the allocation class TYPE. First we try
+ * to satisfy the request from one of the free lists, according
+ * to the free list map provided by the driver. The free list
+ * array has one entry for each request type and the value of
+ * that array element can be one of four possibilities:
*
- * It can be the constant H5FD_MEM_DEFAULT (or zero) which
- * indicates that the identity mapping is used. In other
- * words, the request type maps to its own free list.
+ * It can be the constant H5FD_MEM_DEFAULT (or zero) which
+ * indicates that the identity mapping is used. In other
+ * words, the request type maps to its own free list.
*
- * It can be the request type itself, which has the same
- * effect as the H5FD_MEM_DEFAULT value above.
+ * It can be the request type itself, which has the same
+ * effect as the H5FD_MEM_DEFAULT value above.
*
- * It can be the ID for another request type, which
- * indicates that the free list for the specified type
- * should be used instead.
+ * It can be the ID for another request type, which
+ * indicates that the free list for the specified type
+ * should be used instead.
*
- * It can be the constant H5FD_MEM_NOLIST which means that
- * no free list should be used for this type of request.
+ * It can be the constant H5FD_MEM_NOLIST which means that
+ * no free list should be used for this type of request.
*
- * If the request cannot be satisfied from a free list then
- * either the driver's `alloc' callback is invoked (if one was
- * supplied) or the end-of-address marker is extended. The
- * `alloc' callback is always called with the same arguments as
- * the H5FDalloc().
+ * If the request cannot be satisfied from a free list then
+ * either the driver's `alloc' callback is invoked (if one was
+ * supplied) or the end-of-address marker is extended. The
+ * `alloc' callback is always called with the same arguments as
+ * the H5FDalloc().
*
- * Return: Success: The format address of the new file memory.
+ * Return: Success: The format address of the new file memory.
*
- * Failure: The undefined address HADDR_UNDEF
+ * Failure: The undefined address HADDR_UNDEF
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
*-------------------------------------------------------------------------
@@ -1341,18 +1341,18 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags/*out*/)
haddr_t
H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
{
- haddr_t ret_value = HADDR_UNDEF;
+ haddr_t ret_value = HADDR_UNDEF;
FUNC_ENTER_API(HADDR_UNDEF)
H5TRACE4("a", "*xMtih", file, type, dxpl_id, size);
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid request type")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid request type")
if(size == 0)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "zero-size request")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "zero-size request")
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
else
@@ -1361,7 +1361,7 @@ H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
/* Do the real work */
if(HADDR_UNDEF == (ret_value = H5FD_alloc_real(file, dxpl_id, type, size, NULL, NULL)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate file memory")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate file memory")
/* (Note compensating for base address subtraction in internal routine) */
ret_value += file->base_addr;
@@ -1370,22 +1370,22 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDalloc() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDfree
+ * Function: H5FDfree
*
- * Purpose: Frees format addresses starting with ADDR and continuing for
- * SIZE bytes in the file FILE. The type of space being freed is
- * specified by TYPE, which is mapped to a free list as
- * described for the H5FDalloc() function above. If the request
- * doesn't map to a free list then either the application `free'
- * callback is invoked (if defined) or the memory is leaked.
+ * Purpose: Frees format addresses starting with ADDR and continuing for
+ * SIZE bytes in the file FILE. The type of space being freed is
+ * specified by TYPE, which is mapped to a free list as
+ * described for the H5FDalloc() function above. If the request
+ * doesn't map to a free list then either the application `free'
+ * callback is invoked (if defined) or the memory is leaked.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, July 28, 1999
*
* Modifications:
@@ -1420,17 +1420,17 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDfree() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDget_eoa
+ * Function: H5FDget_eoa
*
- * Purpose: Returns the address of the first byte after the last
- * allocated memory in the file.
+ * Purpose: Returns the address of the first byte after the last
+ * allocated memory in the file.
*
- * Return: Success: First byte after allocated memory.
- * Failure: HADDR_UNDEF
+ * Return: Success: First byte after allocated memory.
+ * Failure: HADDR_UNDEF
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, July 30, 1999
*
*-------------------------------------------------------------------------
@@ -1438,20 +1438,20 @@ done:
haddr_t
H5FDget_eoa(H5FD_t *file, H5FD_mem_t type)
{
- haddr_t ret_value;
+ haddr_t ret_value;
FUNC_ENTER_API(HADDR_UNDEF)
H5TRACE2("a", "*xMt", file, type);
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file type")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file type")
/* The real work */
if(HADDR_UNDEF == (ret_value = H5FD_get_eoa(file, type)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eoa request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eoa request failed")
/* (Note compensating for base address subtraction in internal routine) */
ret_value += file->base_addr;
@@ -1460,27 +1460,27 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDget_eoa() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDset_eoa
+ * Function: H5FDset_eoa
*
- * Purpose: Set the end-of-address marker for the file. The ADDR is the
- * address of the first byte past the last allocated byte of the
- * file. This function is called from two places:
+ * Purpose: Set the end-of-address marker for the file. The ADDR is the
+ * address of the first byte past the last allocated byte of the
+ * file. This function is called from two places:
*
- * It is called after an existing file is opened in order to
- * "allocate" enough space to read the superblock and then
- * to "allocate" the entire hdf5 file based on the contents
- * of the superblock.
+ * It is called after an existing file is opened in order to
+ * "allocate" enough space to read the superblock and then
+ * to "allocate" the entire hdf5 file based on the contents
+ * of the superblock.
*
- * It is called during file memory allocation if the
- * allocation request cannot be satisfied from the free list
- * and the driver didn't supply an allocation callback.
+ * It is called during file memory allocation if the
+ * allocation request cannot be satisfied from the free list
+ * and the driver didn't supply an allocation callback.
*
- * Return: Success: Non-negative
- * Failure: Negative, no side effect
+ * Return: Success: Non-negative
+ * Failure: Negative, no side effect
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, July 30, 1999
*
*-------------------------------------------------------------------------
@@ -1495,42 +1495,42 @@ H5FDset_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr)
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file type")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file type")
if(!H5F_addr_defined(addr) || addr > file->maxaddr)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid end-of-address value")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid end-of-address value")
/* The real work */
/* (Note compensating for base address addition in internal routine) */
if(H5FD_set_eoa(file, type, addr - file->base_addr) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file set eoa request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file set eoa request failed")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDset_eoa() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDget_eof
+ * Function: H5FDget_eof
*
- * Purpose: Returns the end-of-file address, which is the greater of the
- * end-of-format address and the actual EOF marker. This
- * function is called after an existing file is opened in order
- * for the library to learn the true size of the underlying file
- * and to determine whether the hdf5 data has been truncated.
+ * Purpose: Returns the end-of-file address, which is the greater of the
+ * end-of-format address and the actual EOF marker. This
+ * function is called after an existing file is opened in order
+ * for the library to learn the true size of the underlying file
+ * and to determine whether the hdf5 data has been truncated.
*
- * It is also used when a file is first opened to learn whether
- * the file is empty or not.
+ * It is also used when a file is first opened to learn whether
+ * the file is empty or not.
*
- * It is permissible for the driver to return the maximum address
- * for the file size if the file is not empty.
+ * It is permissible for the driver to return the maximum address
+ * for the file size if the file is not empty.
*
- * Return: Success: The EOF address.
+ * Return: Success: The EOF address.
*
- * Failure: HADDR_UNDEF
+ * Failure: HADDR_UNDEF
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
* Modifications:
@@ -1540,18 +1540,18 @@ done:
haddr_t
H5FDget_eof(H5FD_t *file)
{
- haddr_t ret_value;
+ haddr_t ret_value;
FUNC_ENTER_API(HADDR_UNDEF)
H5TRACE1("a", "*x", file);
/* Check arguments */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
/* The real work */
if(HADDR_UNDEF == (ret_value = H5FD_get_eof(file)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eof request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eof request failed")
/* (Note compensating for base address subtraction in internal routine) */
ret_value += file->base_addr;
@@ -1560,16 +1560,16 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDget_eof() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_maxaddr
+ * Function: H5FD_get_maxaddr
*
- * Purpose: Private version of H5FDget_eof()
+ * Purpose: Private version of H5FDget_eof()
*
- * Return: Success: The maximum address allowed in the file.
- * Failure: HADDR_UNDEF
+ * Return: Success: The maximum address allowed in the file.
+ * Failure: HADDR_UNDEF
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 3, 2008
*
*-------------------------------------------------------------------------
@@ -1590,16 +1590,16 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_maxaddr() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_feature_flags
+ * Function: H5FD_get_feature_flags
*
- * Purpose: Retrieve the feature flags for the VFD
+ * Purpose: Retrieve the feature flags for the VFD
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Tuesday, January 8, 2008
*
*-------------------------------------------------------------------------
@@ -1618,16 +1618,16 @@ H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_get_feature_flags() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_fs_type_map
+ * Function: H5FD_get_fs_type_map
*
- * Purpose: Retrieve the free space type mapping for the VFD
+ * Purpose: Retrieve the free space type mapping for the VFD
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 17, 2008
*
*-------------------------------------------------------------------------
@@ -1657,29 +1657,29 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_fs_type_map() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDread
+ * Function: H5FDread
*
- * Purpose: Reads SIZE bytes from FILE beginning at address ADDR
- * according to the data transfer property list DXPL_ID (which may
- * be the constant H5P_DEFAULT). The result is written into the
- * buffer BUF.
+ * Purpose: Reads SIZE bytes from FILE beginning at address ADDR
+ * according to the data transfer property list DXPL_ID (which may
+ * be the constant H5P_DEFAULT). The result is written into the
+ * buffer BUF.
*
- * Return: Success: Non-negative. The read result is written into
- * the BUF buffer which should be allocated by
- * the caller.
+ * Return: Success: Non-negative. The read result is written into
+ * the BUF buffer which should be allocated by
+ * the caller.
*
- * Failure: Negative. The contents of BUF is undefined.
+ * Failure: Negative. The contents of BUF is undefined.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
herr_t
H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
- void *buf/*out*/)
+ void *buf/*out*/)
{
H5P_genplist_t *dxpl; /* DXPL object */
herr_t ret_value = SUCCEED; /* Return value */
@@ -1689,7 +1689,7 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
/* Get the default dataset transfer property list if the user didn't provide one */
if(H5P_DEFAULT == dxpl_id)
@@ -1698,7 +1698,7 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size
if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
if(!buf)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer")
/* Get the DXPL plist object for DXPL ID */
if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
@@ -1707,33 +1707,33 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size
/* Do the real work */
/* (Note compensating for base address addition in internal routine) */
if(H5FD_read(file, dxpl, type, addr - file->base_addr, size, buf) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDread() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDwrite
+ * Function: H5FDwrite
*
- * Purpose: Writes SIZE bytes to FILE beginning at address ADDR according
- * to the data transfer property list DXPL_ID (which may be the
- * constant H5P_DEFAULT). The bytes to be written come from the
- * buffer BUF.
+ * Purpose: Writes SIZE bytes to FILE beginning at address ADDR according
+ * to the data transfer property list DXPL_ID (which may be the
+ * constant H5P_DEFAULT). The bytes to be written come from the
+ * buffer BUF.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
herr_t
H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
- const void *buf)
+ const void *buf)
{
H5P_genplist_t *dxpl; /* DXPL object */
herr_t ret_value = SUCCEED; /* Return value */
@@ -1743,7 +1743,7 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
/* Get the default dataset transfer property list if the user didn't provide one */
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
@@ -1751,7 +1751,7 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz
if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
if(!buf)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer")
/* Get the DXPL plist object for DXPL ID */
if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
@@ -1760,24 +1760,24 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz
/* The real work */
/* (Note compensating for base address addition in internal routine) */
if(H5FD_write(file, dxpl, type, addr - file->base_addr, size, buf) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDwrite() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDflush
+ * Function: H5FDflush
*
- * Purpose: Notify driver to flush all cached data. If the driver has no
- * flush method then nothing happens.
+ * Purpose: Notify driver to flush all cached data. If the driver has no
+ * flush method then nothing happens.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
* Modifications:
@@ -1796,7 +1796,7 @@ H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing)
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
else
@@ -1805,22 +1805,22 @@ H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing)
/* Do the real work */
if(H5FD_flush(file, dxpl_id, closing) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "file flush request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "file flush request failed")
done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_flush
+ * Function: H5FD_flush
*
- * Purpose: Private version of H5FDflush()
+ * Purpose: Private version of H5FDflush()
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
*-------------------------------------------------------------------------
@@ -1841,16 +1841,16 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_flush() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDtruncate
+ * Function: H5FDtruncate
*
- * Purpose: Notify driver to truncate the file back to the allocated size.
+ * Purpose: Notify driver to truncate the file back to the allocated size.
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 31, 2008
*
*-------------------------------------------------------------------------
@@ -1865,7 +1865,7 @@ H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing)
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
else
@@ -1874,22 +1874,22 @@ H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing)
/* Do the real work */
if(H5FD_truncate(file, dxpl_id, closing) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "file flush request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "file flush request failed")
done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_truncate
+ * Function: H5FD_truncate
*
- * Purpose: Private version of H5FDtruncate()
+ * Purpose: Private version of H5FDtruncate()
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 31, 2008
*
*-------------------------------------------------------------------------
@@ -1910,18 +1910,18 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_truncate() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_fileno
+ * Function: H5FD_get_fileno
*
- * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value
+ * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value
* (Mainly added to stop non-file routines from poking about in the
* H5FD_t data structure)
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
- * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu>
- * March 27, 2002
+ * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu>
+ * March 27, 2002
*
*-------------------------------------------------------------------------
*/
@@ -1939,7 +1939,7 @@ H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_get_fileno() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FDget_vfd_handle
*
@@ -1973,7 +1973,7 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDget_vfd_handle() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FD_get_vfd_handle
*
@@ -1998,7 +1998,7 @@ H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle)
HDassert(file_handle);
if(NULL == file->cls->get_handle)
- HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no `get_vfd_handle' method")
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no `get_vfd_handle' method")
if((file->cls->get_handle)(file, fapl, file_handle) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get file handle for file driver")
@@ -2006,7 +2006,7 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_vfd_handle() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FD_set_base_addr
*
@@ -2033,14 +2033,14 @@ H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_set_base_addr() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FD_get_base_addr
*
* Purpose: Get the base address for the file
*
- * Return: Success: The absolute base address of the file
- * Failure: The undefined address (HADDR_UNDEF)
+ * Return: Success: The absolute base address of the file
+ * Failure: The undefined address (HADDR_UNDEF)
*
* Programmer: Quincey Koziol
* Sept. 10, 2009
@@ -2058,3 +2058,37 @@ H5FD_get_base_addr(const H5FD_t *file)
FUNC_LEAVE_NOAPI(file->base_addr)
} /* end H5FD_get_base_addr() */
+/*-------------------------------------------------------------------------
+* Function: H5FDdriver_query
+*
+* Purpose: Similar to H5FD_query(), but intended for cases when we don't
+* have a file available (e.g. before one is opened). Since we
+* can't use the file to get the driver, the driver ID is passed
+* in as a parameter.
+*
+* Return: SUCCEED/FAIL
+*
+*-------------------------------------------------------------------------
+*/
+herr_t
+H5FDdriver_query(hid_t driver_id, unsigned long *flags/*out*/)
+{
+ H5FD_class_t *driver = NULL; /* Pointer to VFD class struct */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "ix", driver_id, flags);
+
+ /* Check arguments */
+ if(NULL == flags)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "flags parameter cannot be NULL")
+
+ /* Check for the driver to query and then query it */
+ if (NULL == (driver = (H5FD_class_t *)H5I_object_verify(driver_id, H5I_VFL)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "not a VFL ID")
+ if (H5FD_driver_query(driver, flags) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "driver flag query failed")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5FDdriver_query() */
diff --git a/src/H5FDcore.h b/src/H5FDcore.h
index 3ee61b6..f2b0fb0 100644
--- a/src/H5FDcore.h
+++ b/src/H5FDcore.h
@@ -15,7 +15,7 @@
* Programmer: Robb Matzke <matzke@llnl.gov>
* Monday, August 2, 1999
*
- * Purpose: The public header file for the sec2 driver.
+ * Purpose: The public header file for the core driver.
*/
#ifndef H5FDcore_H
#define H5FDcore_H
diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c
new file mode 100644
index 0000000..59d954f
--- /dev/null
+++ b/src/H5FDhdfs.c
@@ -0,0 +1,1876 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Read-Only HDFS Virtual File Driver (VFD) *
+ * Copyright (c) 2018, The HDF Group. *
+ * *
+ * All rights reserved. *
+ * *
+ * NOTICE: *
+ * All information contained herein is, and remains, the property of The HDF *
+ * Group. The intellectual and technical concepts contained herein are *
+ * proprietary to The HDF Group. Dissemination of this information or *
+ * reproduction of this material is strictly forbidden unless prior written *
+ * permission is obtained from The HDF Group. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Jacob Smith
+ * 2018-04-23
+ *
+ * Purpose: Provide read-only access to files on the Hadoop Distributed
+ * File System (HDFS).
+ */
+
+/* Interface initialization */
+#define H5_INTERFACE_INIT_FUNC H5FD_hdfs_init_interface
+
+
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* File access */
+#include "H5FDprivate.h" /* File drivers */
+#include "H5FDhdfs.h" /* hdfs file driver */
+#include "H5FLprivate.h" /* Free Lists */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
+
+#ifdef H5_HAVE_LIBHDFS
+#include "hdfs.h"
+#endif
+
+/* toggle function call prints: 1 turns on */
+#define HDFS_DEBUG 0
+
+/* toggle stats collection and reporting */
+#define HDFS_STATS 0
+
+/* The driver identification number, initialized at runtime */
+static hid_t H5FD_HDFS_g = 0;
+
+#if HDFS_STATS
+
+/* arbitrarily large value, such that any reasonable size read will be "less"
+ * than this value and set a true minimum
+ * not 0 because that may be a valid recorded minimum in degenerate cases
+ */
+#define HDFS_STATS_STARTING_MIN 0xfffffffful
+
+/* Configuration definitions for stats collection and breakdown
+ *
+ * 2^10 = 1024
+ * Reads up to 1024 bytes (1 kB) fall in bin 0
+ * 2^(10+(1*16)) = 2^26 = 64MB
+ * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT]
+ */
+#define HDFS_STATS_BASE 2
+#define HDFS_STATS_INTERVAL 1
+#define HDFS_STATS_START_POWER 10
+#define HDFS_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))`
+ * Stores result at `(unsigned long long *) out_ptr`.
+ * Used in computing boundaries between stats bins.
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ */
+#define HDFS_STATS_POW(bin_i, out_ptr) { \
+ unsigned long long donotshadowresult = 1; \
+ unsigned donotshadowindex = 0; \
+ for (donotshadowindex = 0; \
+ donotshadowindex < (((bin_i) * HDFS_STATS_INTERVAL) + \
+ HDFS_STATS_START_POWER); \
+ donotshadowindex++) \
+ { \
+ donotshadowresult *= HDFS_STATS_BASE; \
+ } \
+ *(out_ptr) = donotshadowresult; \
+}
+
+/* array to hold pre-computed boundaries for stats bins */
+static unsigned long long hdfs_stats_boundaries[HDFS_STATS_BIN_COUNT];
+
+
+/***************************************************************************
+ *
+ * Structure: hdfs_statsbin
+ *
+ * Purpose:
+ *
+ * Structure for storing per-file hdfs VFD usage statistics.
+ *
+ *
+ *
+ * `count` (unsigned long long)
+ *
+ * Number of reads with size in this bin's range.
+ *
+ * `bytes` (unsigned long long)
+ *
+ * Total number of bytes read through this bin.
+ *
+ * `min` (unsigned long long)
+ *
+ * Smallest read size in this bin.
+ *
+ * `max` (unsigned long long)
+ *
+ * Largest read size in this bin.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ * Changes: None
+ *
+ ***************************************************************************/
+typedef struct {
+ unsigned long long count;
+ unsigned long long bytes;
+ unsigned long long min;
+ unsigned long long max;
+} hdfs_statsbin;
+
+#endif /* HDFS_STATS */
+
+#ifdef H5_HAVE_LIBHDFS
+
+/* "unique" identifier for `hdfs_t` structures.
+ * Randomly generated by unweighted dice rolls.
+ */
+#define HDFS_HDFST_MAGIC 0x1AD5DE84
+
+
+/***************************************************************************
+ *
+ * Structure: hdfs_t
+ *
+ * Purpose:
+ *
+ * Contain/retain information associated with a file hosted on Hadoop
+ * Distributed File System (HDFS). Instantiated and populated via
+ * `H5FD_hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`.
+ *
+ *
+ *
+ * `magic` (unisgned long)
+ *
+ * Number to indicate that this structure is of the promised
+ * type and should still be valid; should be HDFS_HDFST_MAGIC throughout
+ * the lifespan of the structure. Upon deletion of the structure, the
+ * programmer should set magic to anything but HDFS_HDFST_MAGIC, to
+ * indicate that the structure is to no longer be trusted.
+ *
+ * `filesystem` (hdfsFS)
+ *
+ * A libhdfs file system handle.
+ *
+ * `fileinfo` (hdfsFileInfo*)
+ *
+ * A pointer to a libhdfs file info structure.
+ *
+ * `file` (hdfsFile)
+ *
+ * A libhdfs file handle.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ * May 2018
+ *
+ * Changes: None
+ *
+ ***************************************************************************
+ */
+typedef struct {
+ unsigned long magic;
+ hdfsFS filesystem;
+ hdfsFileInfo *fileinfo;
+ hdfsFile file;
+} hdfs_t;
+
+
+/*--------------------------------------------------------------------------
+ * Function: H5FD_hdfs_handle_open
+ *
+ * Purpose: Create a HDFS file handle, 'opening' the target file.
+ *
+ * Return: Success: Pointer to HDFS container/handle of opened file.
+ * Failure: NULL
+ *
+ * Programmer: Gerd Herber
+ * May 2018
+ *
+ * Changes: None.
+ *--------------------------------------------------------------------------
+ */
+static hdfs_t *
+H5FD_hdfs_handle_open(
+ const char *path,
+ const char *namenode_name,
+ const int32_t namenode_port,
+ const char *user_name,
+ const char *kerberos_ticket_cache,
+ const int32_t stream_buffer_size)
+{
+ struct hdfsBuilder *builder = NULL;
+ hdfs_t *handle = NULL;
+ hdfs_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "called H5FD_hdfs_handle_open.\n");
+#endif
+
+ if (path == NULL || path[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "path cannot be null.\n")
+ }
+ if (namenode_name == NULL /* || namenode_name[0] == '\0' */ ) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "namenode name cannot be null.\n")
+ }
+ if (namenode_port < 0 || namenode_port > 65535) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "namenode port must be non-negative and <= 65535.\n")
+ }
+ if (stream_buffer_size < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "buffer size must non-negative.\n")
+ }
+
+ handle = (hdfs_t *)H5MM_malloc(sizeof(hdfs_t));
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "could not malloc space for handle.\n")
+ }
+
+ handle->magic = (unsigned long)HDFS_HDFST_MAGIC;
+ handle->filesystem = NULL; /* TODO: not a pointer; NULL may cause bug */
+ handle->fileinfo = NULL;
+ handle->file = NULL; /* TODO: not a pointer; NULL may cause bug */
+
+ builder = hdfsNewBuilder();
+ if (!builder) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "(hdfs) failed to create builder")
+ }
+ hdfsBuilderSetNameNode(builder, namenode_name);
+ hdfsBuilderSetNameNodePort(builder, (tPort)namenode_port);
+ if (user_name != NULL && user_name[0] != '\0') {
+ hdfsBuilderSetUserName(builder, user_name);
+ }
+ if (kerberos_ticket_cache != NULL && kerberos_ticket_cache[0] != '\0') {
+ hdfsBuilderSetKerbTicketCachePath(builder, kerberos_ticket_cache);
+ }
+ /* Call to `hdfsBuilderConnect` releases builder, regardless of success. */
+ handle->filesystem = hdfsBuilderConnect(builder);
+ if (!handle->filesystem) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "(hdfs) could not connect to default namenode")
+ }
+ handle->fileinfo = hdfsGetPathInfo(handle->filesystem, path);
+ if (!handle->fileinfo) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "hdfsGetPathInfo failed")
+ }
+ handle->file = hdfsOpenFile(
+ handle->filesystem,
+ path,
+ O_RDONLY,
+ stream_buffer_size,
+ 0,
+ 0);
+ if (!handle->file) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL,
+ "(hdfs) could not open")
+ }
+
+ ret_value = handle;
+
+done:
+ if (ret_value == NULL && handle != NULL) {
+ /* error; clean up */
+ HDassert(handle->magic == HDFS_HDFST_MAGIC);
+ handle->magic++;
+ if (handle->file != NULL) {
+ if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL,
+ "unable to close hdfs file handle")
+ }
+ }
+ if (handle->fileinfo != NULL) {
+ hdfsFreeFileInfo(handle->fileinfo, 1);
+ }
+ if (handle->filesystem != NULL) {
+ if (FAIL == (hdfsDisconnect(handle->filesystem))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL,
+ "unable to disconnect from hdfs")
+ }
+ }
+ H5MM_xfree(handle);
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_hdfs_handle_open() */
+
+
+/*--------------------------------------------------------------------------
+ * Function: H5FD_hdfs_handle_close
+ *
+ * Purpose: 'Close' an HDFS file container/handle, releasing underlying
+ * resources.
+ *
+ * Return: Success: `SUCCEED` (0)
+ * Failure: `FAIL` (-1)
+ *
+ * Programmer: Gerd Herber
+ * May 2018
+ *
+ * Changes: None.
+ *--------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_handle_close(hdfs_t *handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "called H5FD_hdfs_close.\n");
+#endif
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n")
+ }
+ if (handle->magic != HDFS_HDFST_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n")
+ }
+
+ handle->magic++;
+ if (handle->file != NULL) {
+ if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close hdfs file handle")
+ }
+ }
+ if (handle->fileinfo != NULL) {
+ hdfsFreeFileInfo(handle->fileinfo, 1);
+ }
+ if (handle->filesystem != NULL) {
+ if (FAIL == (hdfsDisconnect(handle->filesystem))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL,
+ "unable to disconnect hdfs file system")
+ }
+ }
+
+ H5MM_xfree(handle);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_hdfs_close() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/***************************************************************************
+ *
+ * Structure: H5FD_hdfs_t
+ *
+ * Purpose:
+ *
+ * H5FD_hdfs_t is a structure used to store all information needed to
+ * maintain R/O access to a single HDF5 file in an HDFS file system.
+ * This structure is created when such a file is "opened" and
+ * discarded when it is "closed".
+ *
+ *
+ * `pub` (H5FD_t)
+ *
+ * Instance of H5FD_t which contains all fields common to all VFDs.
+ * It must be the first item in this structure, since at higher levels,
+ * this structure will be treated as an instance of H5FD_t.
+ *
+ * `fa` (H5FD_hdfs_fapl_t)
+ *
+ * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data
+ * needed to "open" the HDF5 file.
+ *
+ * `eoa` (haddr_t)
+ *
+ * End of addressed space in file. After open, it should always
+ * equal the file size.
+ *
+ * `hdfs_handle` (hdfs_t *)
+ *
+ * Instance of HDFS Request handle associated with the target resource.
+ * Responsible for communicating with remote host and presenting file
+ * contents as indistinguishable from a file on the local filesystem.
+ *
+ * *** present only if HDFS_SATS is flagged to enable stats collection ***
+ *
+ * `meta` (hdfs_statsbin[])
+ * `raw` (hdfs_statsbin[])
+ *
+ * Only present if hdfs stats collection is enabled.
+ *
+ * Arrays of `hdfs_statsbin` structures to record raw- and metadata reads.
+ *
+ * Records count and size of reads performed by the VFD, and is used to
+ * print formatted usage statistics to stdout upon VFD shutdown.
+ *
+ * Reads of each raw- and metadata type are recorded in an individual bin
+ * determined by the size of the read. The last bin of each type is
+ * reserved for "big" reads, with no defined upper bound.
+ *
+ * *** end HDFS_STATS ***
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ * Changes: None.
+ *
+ ***************************************************************************
+ */
+typedef struct H5FD_hdfs_t {
+ H5FD_t pub;
+ H5FD_hdfs_fapl_t fa;
+ haddr_t eoa;
+#ifdef H5_HAVE_LIBHDFS
+ hdfs_t *hdfs_handle;
+#endif
+#if HDFS_STATS
+ hdfs_statsbin meta[HDFS_STATS_BIN_COUNT + 1];
+ hdfs_statsbin raw[HDFS_STATS_BIN_COUNT + 1];
+#endif
+} H5FD_hdfs_t;
+
+/*
+ * These macros check for overflow of various quantities. These macros
+ * assume that HDoff_t is signed and haddr_t and size_t are unsigned.
+ *
+ * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t'
+ * is too large to be represented by the second argument
+ * of the file seek function.
+ * Only included if HDFS code should compile.
+ *
+ */
+#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1)
+#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR))
+
+/* Prototypes */
+static void *H5FD_hdfs_fapl_get(H5FD_t *_file);
+static void *H5FD_hdfs_fapl_copy(const void *_old_fa);
+static herr_t H5FD_hdfs_fapl_free(void *_fa);
+static H5FD_t *H5FD_hdfs_open(const char *name, unsigned flags, hid_t fapl_id,
+ haddr_t maxaddr);
+static herr_t H5FD_hdfs_close(H5FD_t *_file);
+static int H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
+static herr_t H5FD_hdfs_query(const H5FD_t *_f1, unsigned long *flags);
+static haddr_t H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
+static haddr_t H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle);
+static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr,
+ size_t size, void *buf);
+static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr,
+ size_t size, const void *buf);
+static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t H5FD_hdfs_lock(H5FD_t *_file, hbool_t rw);
+static herr_t H5FD_hdfs_unlock(H5FD_t *_file);
+static herr_t H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa);
+
+static const H5FD_class_t H5FD_hdfs_g = {
+ "hdfs", /* name */
+ MAXADDR, /* maxaddr */
+ H5F_CLOSE_WEAK, /* fc_degree */
+ NULL, /* sb_size */
+ NULL, /* sb_encode */
+ NULL, /* sb_decode */
+ sizeof(H5FD_hdfs_fapl_t), /* fapl_size */
+ H5FD_hdfs_fapl_get, /* fapl_get */
+ H5FD_hdfs_fapl_copy, /* fapl_copy */
+ H5FD_hdfs_fapl_free, /* fapl_free */
+ 0, /* dxpl_size */
+ NULL, /* dxpl_copy */
+ NULL, /* dxpl_free */
+ H5FD_hdfs_open, /* open */
+ H5FD_hdfs_close, /* close */
+ H5FD_hdfs_cmp, /* cmp */
+ H5FD_hdfs_query, /* query */
+ NULL, /* get_type_map */
+ NULL, /* alloc */
+ NULL, /* free */
+ H5FD_hdfs_get_eoa, /* get_eoa */
+ H5FD_hdfs_set_eoa, /* set_eoa */
+ H5FD_hdfs_get_eof, /* get_eof */
+ H5FD_hdfs_get_handle, /* get_handle */
+ H5FD_hdfs_read, /* read */
+ H5FD_hdfs_write, /* write */
+ NULL, /* flush */
+ H5FD_hdfs_truncate, /* truncate */
+ H5FD_hdfs_lock, /* lock */
+ H5FD_hdfs_unlock, /* unlock */
+ H5FD_FLMAP_DICHOTOMY /* fl_map */
+};
+
+#ifdef H5_HAVE_LIBHDFS
+/* Declare a free list to manage the H5FD_hdfs_t struct */
+H5FL_DEFINE_STATIC(H5FD_hdfs_t);
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_init_interface
+ *
+ * Purpose: Initializes any interface-specific data or routines.
+ *
+ * Return: Success: The driver ID for the hdfs driver.
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_init_interface(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ FUNC_LEAVE_NOAPI(H5FD_hdfs_init())
+} /* H5FD_hdfs_init_interface() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_init
+ *
+ * Purpose: Initialize this driver by registering the driver with the
+ * library.
+ *
+ * Return: Success: The driver ID for the hdfs driver.
+ * Failure: Negative
+ *
+ * Programmer: Robb Matzke
+ * Thursday, July 29, 1999
+ *
+ *-------------------------------------------------------------------------
+ */
+hid_t
+H5FD_hdfs_init(void)
+{
+ hid_t ret_value = H5I_INVALID_HID; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_init() called.\n");
+#endif
+
+ if (H5I_VFL != H5I_get_type(H5FD_HDFS_g))
+ H5FD_HDFS_g = H5FD_register( &H5FD_hdfs_g, sizeof(H5FD_class_t), FALSE);
+
+#if HDFS_STATS
+ /* pre-compute statsbin boundaries
+ */
+ for (unsigned bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) {
+ unsigned long long value = 0;
+ HDFS_STATS_POW(bin_i, &value)
+ hdfs_stats_boundaries[bin_i] = value;
+ }
+#endif
+
+ ret_value = H5FD_HDFS_g;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_init() */
+
+
+/*---------------------------------------------------------------------------
+ * Function: H5FD_log_term
+ *
+ * Purpose: Shut down the VFD
+ *
+ * Returns: <none>
+ *
+ * Programmer: Jacob Smith 2018
+ *
+ *---------------------------------------------------------------------------
+ */
+void
+H5FD_hdfs_term(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_term() called.\n");
+#endif
+
+ /* Reset VFL ID */
+ H5FD_HDFS_g = 0;
+
+ FUNC_LEAVE_NOAPI_VOID
+} /* end H5FD_hdfs_term() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_fapl_hdfs
+ *
+ * Purpose: Modify the file access property list to use the H5FD_HDFS
+ * driver defined in this source file. All driver specfic
+ * properties are passed in as a pointer to a suitably
+ * initialized instance of H5FD_hdfs_fapl_t
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa)
+{
+ H5P_genplist_t *plist = NULL; /* Property list pointer */
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa);
+
+ HDassert(fa != NULL);
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5Pset_fapl_hdfs() called.\n");
+#endif
+
+ if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
+
+ if (FAIL == H5FD_hdfs_validate_config(fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,"invalid hdfs config")
+
+ ret_value = H5P_set_driver(plist, H5FD_HDFS, (void *)fa);
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pset_fapl_hdfs() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_open()
+ *
+ * Purpose: Create and/or opens a file as an HDF5 file.
+ *
+ * Any flag except H5F_ACC_RDONLY will cause an error.
+ *
+ * Return: Success: A pointer to a new file data structure.
+ * The public fields will be initialized by the caller, which is
+ * always H5FD_open().
+ * Failure: NULL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+static H5FD_t *
+H5FD_hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
+{
+ H5FD_hdfs_t *file = NULL;
+ hdfs_t *handle = NULL;
+ H5FD_hdfs_fapl_t fa;
+ H5FD_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_open() called.\n");
+#endif /* HDFS_DEBUG */
+
+ /* Sanity check on file offsets */
+ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t));
+
+ /* Check arguments */
+ if(!path || !*path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name")
+ if(0 == maxaddr || HADDR_UNDEF == maxaddr)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr")
+ if(ADDR_OVERFLOW(maxaddr))
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr")
+
+ if(flags != H5F_ACC_RDONLY)
+ HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed")
+ if(fapl_id == H5P_DEFAULT || fapl_id == H5P_FILE_ACCESS_DEFAULT)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "fapl cannot be H5P_DEFAULT")
+ if(FAIL == H5Pget_fapl_hdfs(fapl_id, &fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get property list")
+
+ handle = H5FD_hdfs_handle_open(path, fa.namenode_name, fa.namenode_port, fa.user_name, fa.kerberos_ticket_cache, fa.stream_buffer_size);
+
+ if(handle == NULL)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open")
+
+ HDassert(handle->magic == HDFS_HDFST_MAGIC);
+
+ /* Create the new file struct */
+ if(NULL == (file = H5FL_CALLOC(H5FD_hdfs_t)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct")
+
+ file->hdfs_handle = handle;
+ HDmemcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t));
+
+#if HDFS_STATS
+ if (FAIL == hdfs_reset_stats(file))
+ HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics")
+#endif /* HDFS_STATS */
+
+ ret_value = (H5FD_t*)file;
+
+done:
+ if(NULL == ret_value) {
+ if (handle != NULL) {
+ if (FAIL == H5FD_hdfs_handle_close(handle))
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close HDFS file handle")
+ }
+ if (file != NULL) {
+ file = H5FL_FREE(H5FD_hdfs_t, file);
+ }
+ } /* end if null return value (error) */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_hdfs_open() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static H5FD_t *
+H5FD_hdfs_open(const char H5_ATTR_UNUSED *path, unsigned H5_ATTR_UNUSED flags, hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED maxaddr)
+{
+ H5FD_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "Illegal open of unsupported virtual file (hdfs)");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_open() */
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_close
+ *
+ * Purpose: Closes an HDF5 file.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL, file not closed.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_close(H5FD_t *_file)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_close() called.\n");
+#endif
+
+ /* Sanity checks */
+ HDassert(file != NULL);
+ HDassert(file->hdfs_handle != NULL);
+ HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+
+ /* Close the underlying request handle */
+ if (file->hdfs_handle != NULL)
+ if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close HDFS file handle")
+ }
+
+#if HDFS_STATS
+ /* TODO: mechanism to re-target stats printout */
+ if (FAIL == hdfs_fprint_stats(stdout, file))
+ HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics")
+#endif /* HDFS_STATS */
+
+ /* Release the file info */
+ file = H5FL_FREE(H5FD_hdfs_t, file);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_close() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_close(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "Illegal close of unsupported Virtual File (hdfs)")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_close() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_cmp()
+ *
+ * Purpose: Compares two files using this driver by their HDFS-provided file info,
+ * field-by-field.
+ *
+ * Return: Equivalent: 0
+ * Not Equivalent: -1
+ *
+ * Programmer: Gerd Herber
+ * May 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static int
+H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
+{
+ const H5FD_hdfs_t *f1 = (const H5FD_hdfs_t *)_f1;
+ const H5FD_hdfs_t *f2 = (const H5FD_hdfs_t *)_f2;
+ hdfsFileInfo *finfo1 = NULL;
+ hdfsFileInfo *finfo2 = NULL;
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_cmp() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HDassert(f1->hdfs_handle != NULL);
+ HDassert(f2->hdfs_handle != NULL);
+ HDassert(f1->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+ HDassert(f2->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+
+ finfo1 = f1->hdfs_handle->fileinfo;
+ finfo2 = f2->hdfs_handle->fileinfo;
+ HDassert(finfo1 != NULL);
+ HDassert(finfo2 != NULL);
+
+ if (finfo1->mKind != finfo2->mKind) { HGOTO_DONE(-1); }
+ if (finfo1->mName != finfo2->mName) { HGOTO_DONE(-1); }
+ if (finfo1->mLastMod != finfo2->mLastMod) { HGOTO_DONE(-1); }
+ if (finfo1->mSize != finfo2->mSize) { HGOTO_DONE(-1); }
+ if (finfo1->mReplication != finfo2->mReplication) { HGOTO_DONE(-1); }
+ if (finfo1->mBlockSize != finfo2->mBlockSize) { HGOTO_DONE(-1); }
+ if (strcmp(finfo1->mOwner, finfo2->mOwner)) { HGOTO_DONE(-1); }
+ if (strcmp(finfo1->mGroup, finfo2->mGroup)) { HGOTO_DONE(-1); }
+ if (finfo1->mPermissions != finfo2->mPermissions) { HGOTO_DONE(-1); }
+ if (finfo1->mLastAccess != finfo2->mLastAccess) { HGOTO_DONE(-1); }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_cmp() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static int
+H5FD_hdfs_cmp(
+ const H5FD_t H5_ATTR_UNUSED *_f1,
+ const H5FD_t H5_ATTR_UNUSED *_f2)
+{
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_cmp() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Note that since the HDFS VFD is read only, most flags
+ * are irrelevant.
+ *
+ * The term "set" is highly misleading...
+ * stores/copies the supported flags in the out-pointer `flags`.
+ *
+ * Return: SUCCEED (Can't fail)
+ *
+ * Programmer: John Mainzer
+ * 9/11/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) /* out variable */
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_query() called.\n");
+#endif
+
+ if(flags) {
+ *flags = 0;
+ *flags |= H5FD_FEAT_DATA_SIEVE;
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5FD_hdfs_query() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_get_eoa()
+ *
+ * Purpose: Gets the end-of-address marker for the file. The EOA marker
+ * is the first address past the last byte allocated in the
+ * format address space.
+ *
+ * Return: The end-of-address marker.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static haddr_t
+H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(file->eoa)
+} /* end H5FD_hdfs_get_eoa() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static haddr_t
+H5FD_hdfs_get_eoa( const H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(0)
+} /* end H5FD_hdfs_get_eoa() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_set_eoa()
+ *
+ * Purpose: Set the end-of-address marker for the file.
+ *
+ * Return: SUCCEED (Can't fail)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n");
+#endif
+
+ file->eoa = addr;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5FD_hdfs_set_eoa() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_set_eoa(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t H5_ATTR_UNUSED addr)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(FAIL)
+} /* H5FD_hdfs_set_eoa() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_get_eof()
+ *
+ * Purpose: Returns the end-of-file marker.
+ *
+ * Return: EOF: the first address past the end of the "file", either the
+ * filesystem file or the HDF5 file.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static haddr_t
+H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n");
+#endif
+
+ HDassert(file->hdfs_handle != NULL);
+ HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+
+ FUNC_LEAVE_NOAPI((size_t) file->hdfs_handle->fileinfo->mSize)
+} /* end H5FD_hdfs_get_eof() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static haddr_t
+H5FD_hdfs_get_eof(const H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI((size_t)0)
+} /* end H5FD_hdfs_get_eof() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_get_handle
+ *
+ * Purpose: Returns the HDFS handle (hdfs_t) of hdfs file driver.
+ *
+ * Returns: SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n");
+#endif /* HDFS_DEBUG */
+
+ if(!file_handle)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid")
+
+ *file_handle = file->hdfs_handle;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_get_handle() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_get_handle(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED fapl, void H5_ATTR_UNUSED **file_handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "Illegal get-handle of unsupported virtual file (hdfs)");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_get_handle() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_read
+ *
+ * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR
+ * into buffer BUF according to data transfer properties in DXPL_ID.
+ *
+ * Return: Success: SUCCEED
+ * - Result is stored in caller-supplied buffer BUF.
+ * Failure: FAIL
+ * - Unable to complete read.
+ * - Contents of buffer `buf` are undefined.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-??
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t addr, size_t size, void *buf /*out*/)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+ herr_t ret_value = SUCCEED;
+ size_t filesize = 0;
+#if HDFS_STATS
+ /* working variables for storing stats */
+ hdfs_statsbin *bin = NULL;
+ unsigned bin_i = 0;
+#endif /* HDFS_STATS */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_read() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HDassert(file != NULL);
+ HDassert(file->hdfs_handle != NULL);
+ HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+ HDassert(buf != NULL);
+
+ filesize = (size_t) file->hdfs_handle->fileinfo->mSize;
+
+ if ((addr > filesize) || ((addr + size) > filesize))
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address")
+
+ if (FAIL == hdfsPread(file->hdfs_handle->filesystem, file->hdfs_handle->file, (tOffset)addr, buf, (tSize)size))
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read")
+
+#if HDFS_STATS
+
+ /* Find which "bin" this read fits in. Can be "overflow" bin.
+ */
+ for (bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) {
+ if ((unsigned long long)size < hdfs_stats_boundaries[bin_i]) {
+ break;
+ }
+ }
+ bin = (type == H5FD_MEM_DRAW)
+ ? &file->raw[bin_i]
+ : &file->meta[bin_i];
+
+ /* Store collected stats in appropriate bin */
+ if (bin->count == 0) {
+ bin->min = size;
+ bin->max = size;
+ }
+ else {
+ if (size < bin->min) { bin->min = size; }
+ if (size > bin->max) { bin->max = size; }
+ }
+ bin->count++;
+ bin->bytes += (unsigned long long)size;
+
+#endif /* HDFS_STATS */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_read() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_read(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, void H5_ATTR_UNUSED *buf)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_read() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL,"Illegal get-handle of unsupported virtual file (hdfs)");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_read() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_write
+ *
+ * Purpose: Write bytes to file.
+ * UNSUPPORTED IN READ-ONLY HDFS VFD.
+ *
+ * Return: FAIL (Not possible with Read-Only S3 file.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_write(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf)
+{
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_write() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot write to read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_write() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_truncate
+ *
+ * Purpose: Makes sure that the true file size is the same (or larger)
+ * than the end-of-address.
+ *
+ * NOT POSSIBLE ON READ-ONLY S3 FILES.
+ *
+ * Return: FAIL (Not possible on Read-Only S3 files.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_truncate(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_truncate() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot truncate read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_truncate() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_lock
+ *
+ * Purpose: Place an advisory lock on a file.
+ * No effect on Read-Only S3 file.
+ *
+ * Suggestion: remove lock/unlock from class
+ * would result in error at H5FD_[un]lock() (H5FD.c)
+ *
+ * Return: SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_lock(
+ H5FD_t H5_ATTR_UNUSED *_file,
+ hbool_t H5_ATTR_UNUSED rw)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD_hdfs_lock() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_unlock
+ *
+ * Purpose: Remove the existing lock on the file.
+ * No effect on Read-Only S3 file.
+ *
+ * Return: SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_unlock(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD_hdfs_unlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_validate_config()
+ *
+ * Purpose: Test to see if the supplied instance of H5FD_hdfs_fapl_t
+ * contains internally consistant data. Return SUCCEED if so,
+ * and FAIL otherwise.
+ *
+ * Note the difference between internally consistant and
+ * correct. As we will have to try to access the target
+ * object to determine whether the supplied data is correct,
+ * we will settle for internal consistancy at this point
+ *
+ * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally
+ * consistant data, FAIL otherwise.
+ *
+ * Programmer: Jacob Smith
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HDassert(fa != NULL);
+
+ if(fa->version != H5FD__CURR_HDFS_FAPL_T_VERSION)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unknown H5FD_hdfs_fapl_t version");
+
+ if(fa->namenode_port > 65535)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number");
+ if(fa->namenode_port < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_validate_config() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_fapl_hdfs
+ *
+ * Purpose: Returns information about the hdfs file access property
+ * list though the function arguments.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: John Mainzer
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out)
+{
+ const H5FD_hdfs_fapl_t *fa = NULL;
+ H5P_genplist_t *plist = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa_out);
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5Pget_fapl_hdfs() called.\n");
+#endif
+
+ if (fa_out == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL")
+ plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+ if (plist == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list")
+ if (H5FD_HDFS != H5P_get_driver(plist))
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
+
+ fa = (const H5FD_hdfs_fapl_t *)H5P_get_driver_info(plist);
+ if (fa == NULL)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info")
+
+ /* Copy the hdfs fapl data out */
+ HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t));
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pget_fapl_hdfs() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_fapl_get
+ *
+ * Purpose: Gets a file access property list which could be used to
+ * create an identical file.
+ *
+ * Return: Success: Ptr to new file access property list value.
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_hdfs_fapl_get(H5FD_t *_file)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t*)_file;
+ H5FD_hdfs_fapl_t *fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t));
+ if(fa == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+
+ /* Copy the fields of the structure */
+ HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t));
+
+ ret_value = fa;
+
+done:
+ if (ret_value == NULL && fa != NULL) {
+ H5MM_xfree(fa); /* clean up on error */
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_fapl_get() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_fapl_copy
+ *
+ * Purpose: Copies the hdfs-specific file access properties.
+ *
+ * Return: Success: Ptr to a new property list
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_hdfs_fapl_copy(const void *_old_fa)
+{
+ const H5FD_hdfs_fapl_t *old_fa = (const H5FD_hdfs_fapl_t*)_old_fa;
+ H5FD_hdfs_fapl_t *new_fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t));
+ if(new_fa == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+
+ HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t));
+ ret_value = new_fa;
+
+done:
+ if (ret_value == NULL && new_fa != NULL)
+ H5MM_xfree(new_fa); /* clean up on error */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_fapl_copy() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_fapl_free
+ *
+ * Purpose: Frees the hdfs-specific file access properties.
+ *
+ * Return: SUCCEED (cannot fail)
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_fapl_free(void *_fa)
+{
+ H5FD_hdfs_fapl_t *fa = (H5FD_hdfs_fapl_t*)_fa;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(fa != NULL); /* sanity check */
+
+ H5MM_xfree(fa);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5FD_hdfs_fapl_free() */
+
+
+#if HDFS_STATS
+
+/*----------------------------------------------------------------------------
+ * Function: hdfs_reset_stats()
+ *
+ * Purpose: Reset the stats collection elements in this virtual file structure.
+ * Clears any set data in stats bins; initializes/zeroes values.
+ *
+ * Return: - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - Occurs if the file is invalid somehow
+ *
+ * Programmer: Jacob Smith
+ * 2017-12-08
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+hdfs_reset_stats(H5FD_hdfs_t *file)
+{
+ unsigned i = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDprintf("hdfs_reset_stats() called\n");
+#endif
+
+ if (file == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null")
+
+ for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) {
+ file->raw[i].bytes = 0;
+ file->raw[i].count = 0;
+ file->raw[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ file->raw[i].max = 0;
+
+ file->meta[i].bytes = 0;
+ file->meta[i].count = 0;
+ file->meta[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ file->meta[i].max = 0;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* hdfs_reset_stats */
+#endif /* HDFS_STATS */
+
+
+#if HDFS_STATS
+
+/*----------------------------------------------------------------------------
+ * Function: hdfs_fprint_stats()
+ *
+ * Purpose: Tabulate and pretty-print statistics for this virtual file.
+ *
+ * Should be called upon file close.
+ *
+ * Shows number of reads and bytes read, broken down by
+ * "raw" (H5FD_MEM_DRAW)
+ * or "meta" (any other flag)
+ *
+ * Prints filename and listing of total number of reads and bytes read,
+ * both as a grand total and separate meta- and rawdata reads.
+ *
+ * If any reads were done, prints out two tables:
+ *
+ * 1. overview of raw- and metadata reads
+ * - min (smallest size read)
+ * - average of size read
+ * - k,M,G suffixes by powers of 1024 (2^10)
+ * - max (largest size read)
+ * 2. tabulation of "bins", sepraring reads into exponentially-larger
+ * ranges of size.
+ * - columns for number of reads, total bytes, and average size, with
+ * separate sub-colums for raw- and metadata reads.
+ * - each row represents one bin, identified by the top of its range
+ *
+ * Bin ranges can be modified with pound-defines at the top of this file.
+ *
+ * Bins without any reads in their bounds are not printed.
+ *
+ * An "overflow" bin is also present, to catch "big" reads.
+ *
+ * Output for all bins (and range ceiling and average size report)
+ * is divied by powers of 1024. By corollary, four digits before the decimal
+ * is valid.
+ *
+ * - 41080 bytes is represented by 40.177k, not 41.080k
+ * - 1004.831M represents approx. 1052642000 bytes
+ *
+ * Return: - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - occurs if the file passed in is invalid
+ * - TODO: if stream is invalid? how can we check this?
+ *
+ * Programmer: Jacob Smith
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+hdfs_fprint_stats(FILE *stream, const H5FD_hdfs_t *file)
+{
+ herr_t ret_value = SUCCEED;
+ parsed_url_t *purl = NULL;
+ unsigned i = 0;
+ unsigned long count_meta = 0;
+ unsigned long count_raw = 0;
+ double average_meta = 0.0;
+ double average_raw = 0.0;
+ unsigned long long min_meta = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ unsigned long long min_raw = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ unsigned long long max_meta = 0;
+ unsigned long long max_raw = 0;
+ unsigned long long bytes_raw = 0;
+ unsigned long long bytes_meta = 0;
+ double re_dub = 0.0; /* re-usable double variable */
+ unsigned suffix_i = 0;
+ const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' };
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ if (stream == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file stream cannot be null" )
+ if (file == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file cannot be null")
+ if (file->hdfs_handle == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle cannot be null")
+ if (file->hdfs_handle->magic != HDFS_HDFST_MAGIC)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle has invalid magic")
+
+ /*******************
+ * AGGREGATE STATS *
+ *******************/
+
+ for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) {
+ const hdfs_statsbin *r = &file->raw[i];
+ const hdfs_statsbin *m = &file->meta[i];
+
+ if (m->min < min_meta) { min_meta = m->min; }
+ if (r->min < min_raw) { min_raw = r->min; }
+ if (m->max > max_meta) { max_meta = m->max; }
+ if (r->max > max_raw) { max_raw = r->max; }
+
+ count_raw += r->count;
+ count_meta += m->count;
+ bytes_raw += r->bytes;
+ bytes_meta += m->bytes;
+ }
+ if (count_raw > 0) {
+ average_raw = (double)bytes_raw / (double)count_raw;
+ }
+ if (count_meta > 0) {
+ average_meta = (double)bytes_meta / (double)count_meta;
+ }
+
+ /******************
+ * PRINT OVERVIEW *
+ ******************/
+
+ HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n",
+ count_raw + count_meta, count_meta, count_raw);
+ HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n",
+ bytes_raw + bytes_meta, bytes_meta, bytes_raw);
+
+ if (count_raw + count_meta == 0) {
+ goto done;
+ }
+
+ /*************************
+ * PRINT AGGREGATE STATS *
+ *************************/
+
+ HDfprintf(stream, "SIZES meta raw\n");
+ HDfprintf(stream, " min ");
+ if (count_meta == 0) {
+ HDfprintf(stream, " 0.000 ");
+ }
+ else {
+ re_dub = (double)min_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+ }
+
+ if (count_raw == 0) {
+ HDfprintf(stream, " 0.000 \n");
+ }
+ else {
+ re_dub = (double)min_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+ }
+
+ HDfprintf(stream, " avg ");
+ re_dub = (double)average_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)average_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ HDfprintf(stream, " max ");
+ re_dub = (double)max_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)max_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ /******************************
+ * PRINT INDIVIDUAL BIN STATS *
+ ******************************/
+
+ HDfprintf(stream,
+ "BINS # of reads total bytes average size\n");
+ HDfprintf(stream,
+ " up-to meta raw meta raw meta raw\n");
+
+ for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) {
+ const hdfs_statsbin *m;
+ const hdfs_statsbin *r;
+ unsigned long long range_end = 0;
+ char bm_suffix = ' '; /* bytes-meta */
+ double bm_val = 0.0;
+ char br_suffix = ' '; /* bytes-raw */
+ double br_val = 0.0;
+ char am_suffix = ' '; /* average-meta */
+ double am_val = 0.0;
+ char ar_suffix = ' '; /* average-raw */
+ double ar_val = 0.0;
+
+ m = &file->meta[i];
+ r = &file->raw[i];
+ if (r->count == 0 && m->count == 0) {
+ continue;
+ }
+
+ range_end = hdfs_stats_boundaries[i];
+
+ if (i == HDFS_STATS_BIN_COUNT) {
+ range_end = hdfs_stats_boundaries[i-1];
+ HDfprintf(stream, ">");
+ } else {
+ HDfprintf(stream, " ");
+ }
+
+ bm_val = (double)m->bytes;
+ for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) {
+ bm_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ bm_suffix = suffixes[suffix_i];
+
+ br_val = (double)r->bytes;
+ for (suffix_i = 0; br_val >= 1024.0; suffix_i++) {
+ br_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ br_suffix = suffixes[suffix_i];
+
+ if (m->count > 0) {
+ am_val = (double)(m->bytes) / (double)(m->count);
+ }
+ for (suffix_i = 0; am_val >= 1024.0; suffix_i++) {
+ am_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ am_suffix = suffixes[suffix_i];
+
+ if (r->count > 0) {
+ ar_val = (double)(r->bytes) / (double)(r->count);
+ }
+ for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) {
+ ar_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ ar_suffix = suffixes[suffix_i];
+
+ re_dub = (double)range_end;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+
+ HDfprintf(
+ stream,
+ " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n",
+ re_dub, suffixes[suffix_i], /* bin ceiling */
+ m->count, /* metadata reads */
+ r->count, /* rawdata reads */
+ bm_val, bm_suffix, /* metadata bytes */
+ br_val, br_suffix, /* rawdata bytes */
+ am_val, am_suffix, /* metadata average */
+ ar_val, ar_suffix); /* rawdata average */
+ fflush(stream);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* hdfs_fprint_stats */
+#endif /* HDFS_STATS */
+
diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h
new file mode 100644
index 0000000..328d36a
--- /dev/null
+++ b/src/H5FDhdfs.h
@@ -0,0 +1,123 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Read-Only HDFS Virtual File Driver (VFD) *
+ * Copyright (c) 2018, The HDF Group. *
+ * *
+ * All rights reserved. *
+ * *
+ * NOTICE: *
+ * All information contained herein is, and remains, the property of The HDF *
+ * Group. The intellectual and technical concepts contained herein are *
+ * proprietary to The HDF Group. Dissemination of this information or *
+ * reproduction of this material is strictly forbidden unless prior written *
+ * permission is obtained from The HDF Group. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Jacob Smith
+ * 2018-04-23
+ *
+ * Purpose: The public header file for the hdfs driver.
+ */
+
+#ifndef H5FDhdfs_H
+#define H5FDhdfs_H
+
+#define H5FD_HDFS (H5FD_hdfs_init())
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/****************************************************************************
+ *
+ * Structure: H5FD_hdfs_fapl_t
+ *
+ * Purpose:
+ *
+ * H5FD_hdfs_fapl_t is a public structure that is used to pass
+ * configuration information to the appropriate HDFS VFD via the FAPL.
+ * A pointer to an instance of this structure is a parameter to
+ * H5Pset_fapl_hdfs() and H5Pget_fapl_hdfs().
+ *
+ *
+ *
+ * `version` (int32_t)
+ *
+ * Version number of the `H5FD_hdfs_fapl_t` structure. Any instance passed
+ * to the above calls must have a recognized version number, or an error
+ * will be flagged.
+ *
+ * This field should be set to `H5FD__CURR_HDFS_FAPL_T_VERSION`.
+ *
+ * `namenode_name` (const char[])
+ *
+ * Name of "Name Node" to access as the HDFS server.
+ *
+ * Must not be longer than `H5FD__HDFS_NODE_NAME_SPACE`.
+ *
+ * TBD: Can be NULL.
+ *
+ * `namenode_port` (int32_t) TBD
+ *
+ * Port number to use to connect with Name Node.
+ *
+ * TBD: If 0, uses a default port.
+ *
+ * `kerberos_ticket_cache` (const char[])
+ *
+ * Path to the location of the Kerberos authentication cache.
+ *
+ * Must not be longer than `H5FD__HDFS_KERB_CACHE_PATH_SPACE`.
+ *
+ * TBD: Can be NULL.
+ *
+ * `user_name` (const char[])
+ *
+ * Username to use when accessing file.
+ *
+ * Must not be longer than `H5FD__HDFS_USER_NAME_SPACE`.
+ *
+ * TBD: Can be NULL.
+ *
+ * `stream_buffer_size` (int32_t)
+ *
+ * Size (in bytes) of the file read stream buffer.
+ *
+ * TBD: If -1, relies on a default value.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ * 2018-04-23
+ *
+ * Changes: None
+ *
+ ****************************************************************************/
+
+#define H5FD__CURR_HDFS_FAPL_T_VERSION 1
+
+#define H5FD__HDFS_NODE_NAME_SPACE 128
+#define H5FD__HDFS_USER_NAME_SPACE 128
+#define H5FD__HDFS_KERB_CACHE_PATH_SPACE 128
+
+typedef struct H5FD_hdfs_fapl_t {
+ int32_t version;
+ char namenode_name[H5FD__HDFS_NODE_NAME_SPACE + 1];
+ int32_t namenode_port;
+ char user_name[H5FD__HDFS_USER_NAME_SPACE + 1];
+ char kerberos_ticket_cache[H5FD__HDFS_KERB_CACHE_PATH_SPACE + 1];
+ int32_t stream_buffer_size;
+} H5FD_hdfs_fapl_t;
+
+H5_DLL hid_t H5FD_hdfs_init(void);
+H5_DLL void H5FD_hdfs_term(void);
+H5_DLL herr_t H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out);
+H5_DLL herr_t H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef H5FDhdfs_H */
+
+
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index 9568f46..8c7f414 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -22,13 +22,13 @@
#include "H5FDpublic.h"
/* Private headers needed by this file */
-#include "H5Pprivate.h" /* Property lists */
+#include "H5Pprivate.h" /* Property lists */
/*
* The MPI drivers are needed because there are
* places where we check for things that aren't handled by these drivers.
*/
-#include "H5FDmpi.h" /* MPI-based file drivers */
+#include "H5FDmpi.h" /* MPI-based file drivers */
/**************************/
@@ -88,6 +88,12 @@ typedef struct {
} \
}
+/* Define structure to hold driver ID & info for FAPLs */
+typedef struct {
+ hid_t driver_id; /* Driver's ID */
+ const void *driver_info; /* Driver info, for open callbacks */
+} H5FD_driver_prop_t;
+
/*****************************/
/* Library Private Variables */
@@ -113,7 +119,7 @@ H5_DLL herr_t H5FD_fapl_open(struct H5P_genplist_t *plist, hid_t driver_id, cons
H5_DLL herr_t H5FD_fapl_close(hid_t driver_id, void *fapl);
H5_DLL hid_t H5FD_register(const void *cls, size_t size, hbool_t app_ref);
H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id,
- haddr_t maxaddr);
+ haddr_t maxaddr);
H5_DLL herr_t H5FD_close(H5FD_t *file);
H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2);
H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f,
@@ -134,6 +140,8 @@ H5_DLL herr_t H5FD_write(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t ty
haddr_t addr, size_t size, const void *buf);
H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, unsigned closing);
H5_DLL herr_t H5FD_truncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
+H5_DLL herr_t H5FD_lock(H5FD_t *file, hbool_t rw);
+H5_DLL herr_t H5FD_unlock(H5FD_t *file);
H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum);
H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle);
H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr);
@@ -145,7 +153,7 @@ H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file);
H5_DLL haddr_t H5FD_mpi_MPIOff_to_haddr(MPI_Offset mpi_off);
H5_DLL herr_t H5FD_mpi_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off/*out*/);
H5_DLL herr_t H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info,
- MPI_Comm *comm_new, MPI_Info *info_new);
+ MPI_Comm *comm_new, MPI_Info *info_new);
H5_DLL herr_t H5FD_mpi_comm_info_free(MPI_Comm *comm, MPI_Info *info);
#ifdef NOT_YET
H5_DLL herr_t H5FD_mpio_wait_for_left_neighbor(H5FD_t *file);
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index a52c57a..3faaf33 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -19,13 +19,13 @@
#define _H5FDpublic_H
#include "H5public.h"
-#include "H5Fpublic.h" /*for H5F_close_degree_t */
+#include "H5Fpublic.h" /*for H5F_close_degree_t */
#define H5_HAVE_VFL 1 /*define a convenient app feature test*/
#define H5FD_VFD_DEFAULT 0 /* Default VFL driver value */
/* Types of allocation requests: see H5Fpublic.h */
-typedef enum H5F_mem_t H5FD_mem_t;
+typedef enum H5F_mem_t H5FD_mem_t;
/* Map "fractal heap" header blocks to 'ohdr' type file memory, since its
* a fair amount of work to add a new kind of file memory and they are similar
@@ -79,42 +79,42 @@ typedef enum H5F_mem_t H5FD_mem_t;
* want to make most efficient reuse of freed memory. The use of the
* H5FD_MEM_SUPER free list is arbitrary.
*/
-#define H5FD_FLMAP_SINGLE { \
- H5FD_MEM_SUPER, /*default*/ \
- H5FD_MEM_SUPER, /*super*/ \
- H5FD_MEM_SUPER, /*btree*/ \
- H5FD_MEM_SUPER, /*draw*/ \
- H5FD_MEM_SUPER, /*gheap*/ \
- H5FD_MEM_SUPER, /*lheap*/ \
- H5FD_MEM_SUPER /*ohdr*/ \
+#define H5FD_FLMAP_SINGLE { \
+ H5FD_MEM_SUPER, /*default*/ \
+ H5FD_MEM_SUPER, /*super*/ \
+ H5FD_MEM_SUPER, /*btree*/ \
+ H5FD_MEM_SUPER, /*draw*/ \
+ H5FD_MEM_SUPER, /*gheap*/ \
+ H5FD_MEM_SUPER, /*lheap*/ \
+ H5FD_MEM_SUPER /*ohdr*/ \
}
/*
* A free-list map which segregates requests into `raw' or `meta' data
* pools.
*/
-#define H5FD_FLMAP_DICHOTOMY { \
- H5FD_MEM_SUPER, /*default*/ \
- H5FD_MEM_SUPER, /*super*/ \
- H5FD_MEM_SUPER, /*btree*/ \
- H5FD_MEM_DRAW, /*draw*/ \
- H5FD_MEM_DRAW, /*gheap*/ \
- H5FD_MEM_SUPER, /*lheap*/ \
- H5FD_MEM_SUPER /*ohdr*/ \
+#define H5FD_FLMAP_DICHOTOMY { \
+ H5FD_MEM_SUPER, /*default*/ \
+ H5FD_MEM_SUPER, /*super*/ \
+ H5FD_MEM_SUPER, /*btree*/ \
+ H5FD_MEM_DRAW, /*draw*/ \
+ H5FD_MEM_DRAW, /*gheap*/ \
+ H5FD_MEM_SUPER, /*lheap*/ \
+ H5FD_MEM_SUPER /*ohdr*/ \
}
/*
* The default free list map which causes each request type to use it's own
* free-list.
*/
-#define H5FD_FLMAP_DEFAULT { \
- H5FD_MEM_DEFAULT, /*default*/ \
- H5FD_MEM_DEFAULT, /*super*/ \
- H5FD_MEM_DEFAULT, /*btree*/ \
- H5FD_MEM_DEFAULT, /*draw*/ \
- H5FD_MEM_DEFAULT, /*gheap*/ \
- H5FD_MEM_DEFAULT, /*lheap*/ \
- H5FD_MEM_DEFAULT /*ohdr*/ \
+#define H5FD_FLMAP_DEFAULT { \
+ H5FD_MEM_DEFAULT, /*default*/ \
+ H5FD_MEM_DEFAULT, /*super*/ \
+ H5FD_MEM_DEFAULT, /*btree*/ \
+ H5FD_MEM_DEFAULT, /*draw*/ \
+ H5FD_MEM_DEFAULT, /*gheap*/ \
+ H5FD_MEM_DEFAULT, /*lheap*/ \
+ H5FD_MEM_DEFAULT /*ohdr*/ \
}
@@ -187,7 +187,7 @@ typedef enum H5F_mem_t H5FD_mem_t;
* instead of the default H5D_ALLOC_TIME_LATE
*/
#define H5FD_FEAT_ALLOCATE_EARLY 0x00000200
- /*
+ /*
* Defining the H5FD_FEAT_ALLOW_FILE_IMAGE for a VFL driver means that
* the driver is able to use a file image in the fapl as the initial
* contents of a file.
@@ -245,9 +245,9 @@ typedef struct H5FD_class_t {
/* A free list is a singly-linked list of address/size pairs. */
typedef struct H5FD_free_t {
- haddr_t addr;
- hsize_t size;
- struct H5FD_free_t *next;
+ haddr_t addr;
+ hsize_t size;
+ struct H5FD_free_t *next;
} H5FD_free_t;
/*
@@ -270,7 +270,7 @@ struct H5FD_t {
/* Define enum for the source of file image callbacks */
typedef enum {
H5FD_FILE_IMAGE_OP_NO_OP,
- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
+ H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY,
H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET,
H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE,
@@ -281,13 +281,13 @@ typedef enum {
/* Define structure to hold file image callbacks */
typedef struct {
- void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op,
+ void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op,
void *udata);
void *(*image_memcpy)(void *dest, const void *src, size_t size,
H5FD_file_image_op_t file_image_op, void *udata);
- void *(*image_realloc)(void *ptr, size_t size,
+ void *(*image_realloc)(void *ptr, size_t size,
H5FD_file_image_op_t file_image_op, void *udata);
- herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op,
+ herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op,
void *udata);
void *(*udata_copy)(void *udata);
herr_t (*udata_free)(void *udata);
@@ -319,6 +319,11 @@ H5_DLL herr_t H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id,
haddr_t addr, size_t size, const void *buf);
H5_DLL herr_t H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing);
H5_DLL herr_t H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
+/* H5_DLL herr_t H5FDlock(H5FD_t *file, hbool_t rw);
+H5_DLL herr_t H5FDunlock(H5FD_t *file); */
+/* Allows querying a VFD ID for features before the file is opened */
+H5_DLL herr_t H5FDdriver_query(hid_t driver_id, unsigned long *flags/*out*/);
+
#ifdef __cplusplus
}
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
new file mode 100644
index 0000000..5456a1b
--- /dev/null
+++ b/src/H5FDros3.c
@@ -0,0 +1,1787 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Programmer: Jacob Smith <jake.smith@hdfgroup.org>
+ * 2017-10-13
+ *
+ * Purpose: Provide read-only access to files hosted on Amazon's S3 service.
+ * Relies on "s3comms" utility layer to implement the AWS REST API.
+ */
+
+/* Interface initialization */
+#define H5_INTERFACE_INIT_FUNC H5FD_ros3_init_interface
+
+
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* File access */
+#include "H5FDprivate.h" /* File drivers */
+#include "H5FDros3.h" /* ros3 file driver */
+#include "H5FLprivate.h" /* Free Lists */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
+#include "H5FDs3comms.h" /* S3 Communications */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/* toggle function call prints: 1 turns on
+ */
+#define ROS3_DEBUG 0
+
+/* toggle stats collection and reporting
+ */
+#define ROS3_STATS 0
+
+/* The driver identification number, initialized at runtime
+ */
+static hid_t H5FD_ROS3_g = 0;
+
+#if ROS3_STATS
+
+/* arbitrarily large value, such that any reasonable size read will be "less"
+ * than this value and set a true minimum
+ * not 0 because that may be a valid recorded minimum in degenerate cases
+ */
+#define ROS3_STATS_STARTING_MIN 0xfffffffful
+
+/* Configuration definitions for stats collection and breakdown
+ *
+ * 2^10 = 1024
+ * Reads up to 1024 bytes (1 kB) fall in bin 0
+ * 2^(10+(1*16)) = 2^26 = 64MB
+ * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT]
+ */
+#define ROS3_STATS_BASE 2
+#define ROS3_STATS_INTERVAL 1
+#define ROS3_STATS_START_POWER 10
+#define ROS3_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */
+
+
+/*
+ * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))`
+ * Stores result at `(unsigned long long *) out_ptr`.
+ * Used in computing boundaries between stats bins.
+ */
+#define ROS3_STATS_POW(bin_i, out_ptr) { \
+ unsigned long long donotshadowresult = 1; \
+ unsigned donotshadowindex = 0; \
+ for (donotshadowindex = 0; \
+ donotshadowindex < (((bin_i) * ROS3_STATS_INTERVAL) + \
+ ROS3_STATS_START_POWER); \
+ donotshadowindex++) \
+ { \
+ donotshadowresult *= ROS3_STATS_BASE; \
+ } \
+ *(out_ptr) = donotshadowresult; \
+}
+
+/* array to hold pre-computed boundaries for stats bins
+ */
+static unsigned long long ros3_stats_boundaries[ROS3_STATS_BIN_COUNT];
+
+/***************************************************************************
+ *
+ * Structure: ros3_statsbin
+ *
+ * Purpose:
+ *
+ * Structure for storing per-file ros3 VFD usage statistics.
+ *
+ *
+ *
+ * `count` (unsigned long long)
+ *
+ * Number of reads with size in this bin's range.
+ *
+ * `bytes` (unsigned long long)
+ *
+ * Total number of bytes read through this bin.
+ *
+ * `min` (unsigned long long)
+ *
+ * Smallest read size in this bin.
+ *
+ * `max` (unsigned long long)
+ *
+ * Largest read size in this bin.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ ***************************************************************************/
+typedef struct {
+ unsigned long long count;
+ unsigned long long bytes;
+ unsigned long long min;
+ unsigned long long max;
+} ros3_statsbin;
+
+#endif /* ROS3_STATS */
+
+/***************************************************************************
+ *
+ * Structure: H5FD_ros3_t
+ *
+ * Purpose:
+ *
+ * H5FD_ros3_t is a structure used to store all information needed to
+ * maintain R/O access to a single HDF5 file that has been stored as a
+ * S3 object. This structure is created when such a file is "opened" and
+ * discarded when it is "closed".
+ *
+ * Presents an S3 object as a file to the HDF5 library.
+ *
+ *
+ *
+ * `pub` (H5FD_t)
+ *
+ * Instance of H5FD_t which contains all fields common to all VFDs.
+ * It must be the first item in this structure, since at higher levels,
+ * this structure will be treated as an instance of H5FD_t.
+ *
+ * `fa` (H5FD_ros3_fapl_t)
+ *
+ * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data
+ * needed to "open" the HDF5 file.
+ *
+ * `eoa` (haddr_t)
+ *
+ * End of addressed space in file. After open, it should always
+ * equal the file size.
+ *
+ * `s3r_handle` (s3r_t *)
+ *
+ * Instance of S3 Request handle associated with the target resource.
+ * Responsible for communicating with remote host and presenting file
+ * contents as indistinguishable from a file on the local filesystem.
+ *
+ * *** present only if ROS3_SATS is flagged to enable stats collection ***
+ *
+ * `meta` (ros3_statsbin[])
+ * `raw` (ros3_statsbin[])
+ *
+ * Only present if ros3 stats collection is enabled.
+ *
+ * Arrays of `ros3_statsbin` structures to record raw- and metadata reads.
+ *
+ * Records count and size of reads performed by the VFD, and is used to
+ * print formatted usage statistics to stdout upon VFD shutdown.
+ *
+ * Reads of each raw- and metadata type are recorded in an individual bin
+ * determined by the size of the read. The last bin of each type is
+ * reserved for "big" reads, with no defined upper bound.
+ *
+ * *** end ROS3_STATS ***
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ ***************************************************************************/
+typedef struct H5FD_ros3_t {
+ H5FD_t pub;
+ H5FD_ros3_fapl_t fa;
+ haddr_t eoa;
+ s3r_t *s3r_handle;
+#if ROS3_STATS
+ ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1];
+ ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1];
+#endif
+} H5FD_ros3_t;
+
+/*
+ * These macros check for overflow of various quantities. These macros
+ * assume that HDoff_t is signed and haddr_t and size_t are unsigned.
+ *
+ * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t'
+ * is too large to be represented by the second argument
+ * of the file seek function.
+ * Only included if it may be used -- ROS3 VFD is enabled.
+ *
+ */
+#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1)
+#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR))
+
+/* Prototypes */
+static H5FD_t *H5FD_ros3_open(const char *name, unsigned flags, hid_t fapl_id,
+ haddr_t maxaddr);
+static herr_t H5FD_ros3_close(H5FD_t *_file);
+static int H5FD_ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
+static herr_t H5FD_ros3_query(const H5FD_t *_f1, unsigned long *flags);
+static haddr_t H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_ros3_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
+static haddr_t H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle);
+static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id,
+ haddr_t addr, size_t size, void *buf);
+static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id,
+ haddr_t addr, size_t size, const void *buf);
+static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t H5FD_ros3_lock(H5FD_t *_file, hbool_t rw);
+static herr_t H5FD_ros3_unlock(H5FD_t *_file);
+static herr_t H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa);
+static void *H5FD_ros3_fapl_get(H5FD_t *_file);
+static void *H5FD_ros3_fapl_copy(const void *_old_fa);
+static herr_t H5FD_ros3_fapl_free(void *_fa);
+
+static const H5FD_class_t H5FD_ros3_g = {
+ "ros3", /* name */
+ MAXADDR, /* maxaddr */
+ H5F_CLOSE_WEAK, /* fc_degree */
+ H5FD_ros3_term, /* terminate */
+ NULL, /* sb_size */
+ NULL, /* sb_encode */
+ NULL, /* sb_decode */
+ sizeof(H5FD_ros3_fapl_t), /* fapl_size */
+ H5FD_ros3_fapl_get, /* fapl_get */
+ H5FD_ros3_fapl_copy, /* fapl_copy */
+ H5FD_ros3_fapl_free, /* fapl_free */
+ 0, /* dxpl_size */
+ NULL, /* dxpl_copy */
+ NULL, /* dxpl_free */
+ H5FD_ros3_open, /* open */
+ H5FD_ros3_close, /* close */
+ H5FD_ros3_cmp, /* cmp */
+ H5FD_ros3_query, /* query */
+ NULL, /* get_type_map */
+ NULL, /* alloc */
+ NULL, /* free */
+ H5FD_ros3_get_eoa, /* get_eoa */
+ H5FD_ros3_set_eoa, /* set_eoa */
+ H5FD_ros3_get_eof, /* get_eof */
+ H5FD_ros3_get_handle, /* get_handle */
+ H5FD_ros3_read, /* read */
+ H5FD_ros3_write, /* write */
+ NULL, /* flush */
+ H5FD_ros3_truncate, /* truncate */
+ H5FD_ros3_lock, /* lock */
+ H5FD_ros3_unlock, /* unlock */
+ H5FD_FLMAP_DICHOTOMY /* fl_map */
+};
+
+/* Declare a free list to manage the H5FD_ros3_t struct */
+H5FL_DEFINE_STATIC(H5FD_ros3_t);
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_init_interface
+ *
+ * Purpose: Initializes any interface-specific data or routines.
+ *
+ * Return: Success: The driver ID for the ros3 driver.
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_init_interface(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ FUNC_LEAVE_NOAPI(H5FD_ros3_init())
+} /* H5FD_ros3_init_interface() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_init
+ *
+ * Purpose: Initialize this driver by registering the driver with the
+ * library.
+ *
+ * Return: Success: The driver ID for the ros3 driver.
+ * Failure: Negative
+ *
+ * Programmer: Jacob Smith 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+hid_t
+H5FD_ros3_init(void)
+{
+ hid_t ret_value = H5I_INVALID_HID;
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_init() called.\n");
+#endif
+
+ if (H5I_VFL != H5I_get_type(H5FD_ROS3_g))
+ H5FD_ROS3_g = H5FD_register(&H5FD_ros3_g, sizeof(H5FD_class_t), FALSE);
+
+#if ROS3_STATS
+ /* pre-compute statsbin boundaries
+ */
+ for (unsigned bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) {
+ unsigned long long value = 0;
+ ROS3_STATS_POW(bin_i, &value)
+ ros3_stats_boundaries[bin_i] = value;
+ }
+#endif
+
+ /* Set return value */
+ ret_value = H5FD_ROS3_g;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_ros3_init() */
+
+
+/*---------------------------------------------------------------------------
+ * Function: H5FD_ros3_term
+ *
+ * Purpose: Shut down the VFD
+ *
+ * Returns: SUCCEED (Can't fail)
+ *
+ * Programmer: Jacob Smith 2017
+ *
+ *---------------------------------------------------------------------------
+ */
+void
+H5FD_ros3_term(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_term() called.\n");
+#endif
+
+ /* Reset VFL ID */
+ H5FD_ROS3_g = 0;
+
+ FUNC_LEAVE_NOAPI_VOID
+} /* end H5FD_ros3_term() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_fapl_ros3
+ *
+ * Purpose: Modify the file access property list to use the H5FD_ROS3
+ * driver defined in this source file. All driver specfic
+ * properties are passed in as a pointer to a suitably
+ * initialized instance of H5FD_ros3_fapl_t
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: John Mainzer
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa)
+{
+ H5P_genplist_t *plist = NULL; /* Property list pointer */
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa);
+
+ HDassert(fa != NULL);
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5Pset_fapl_ros3() called.\n");
+#endif
+
+ if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
+
+ if (FAIL == H5FD_ros3_validate_config(fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config")
+
+ ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa);
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5Pset_fapl_ros3() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_open()
+ *
+ * Purpose: Create and/or opens a file as an HDF5 file.
+ * Any flag except H5F_ACC_RDONLY will cause an error.
+ *
+ * Name (as received from `H5FD_open()`) must conform to web url:
+ * NAME :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ]
+ * HTTP :: "http" [ "s" ]
+ * DOMAIN :: e.g., "mybucket.host.org"
+ * PORT :: ":" <number> (e.g., ":9000" )
+ * URI :: <string> (e.g., "path/to/resource.hd5" )
+ * QUERY :: "?" <string> (e.g., "arg1=param1&arg2=param2")
+ *
+ * Return: Success: A pointer to a new file data structure.
+ * The public fields will be initialized by the caller, which is
+ * always H5FD_open().
+ * Failure: NULL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static H5FD_t *
+H5FD_ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
+{
+ H5FD_ros3_t *file = NULL;
+ struct tm *now = NULL;
+ char iso8601now[ISO8601_SIZE];
+ unsigned char signing_key[SHA256_DIGEST_LENGTH];
+ s3r_t *handle = NULL;
+ H5FD_ros3_fapl_t fa;
+ H5FD_t *ret_value = NULL;
+
+
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_open() called.\n");
+#endif
+
+ /* Sanity check on file offsets */
+ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t));
+
+ /* Check arguments */
+ if(!url || !*url)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name")
+ if(0 == maxaddr || HADDR_UNDEF == maxaddr)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr")
+ if(ADDR_OVERFLOW(maxaddr))
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr")
+ if(flags != H5F_ACC_RDONLY)
+ HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed")
+
+ if (FAIL == H5Pget_fapl_ros3(fapl_id, &fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get property list")
+
+ if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)")
+
+ /* open file; procedure depends on whether or not the fapl instructs to
+ * authenticate requests or not.
+ */
+ if (fa.authenticate == TRUE) {
+ /* compute signing key (part of AWS/S3 REST API)
+ * can be re-used by user/key for 7 days after creation.
+ * find way to re-use/share
+ */
+ now = gmnow();
+ HDassert( now != NULL );
+ if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while writing iso8601 timestamp")
+ if (FAIL == H5FD_s3comms_signing_key(signing_key,
+ (const char *)fa.secret_key,
+ (const char *)fa.aws_region,
+ (const char *)iso8601now) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key")
+ }
+
+ handle = H5FD_s3comms_s3r_open(
+ url,
+ (const char *)fa.aws_region,
+ (const char *)fa.secret_id,
+ (const unsigned char *)signing_key);
+ }
+ else {
+ handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL);
+ } /* if/else should authenticate */
+
+ if (handle == NULL) {
+ /* If we want to check CURL's say on the matter in a controlled
+ * fashion, this is the place to do it, but would need to make a
+ * few minor changes to s3comms `s3r_t` and `s3r_read()`.
+ */
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open");
+ }
+
+ /* create new file struct
+ */
+ file = H5FL_CALLOC(H5FD_ros3_t);
+ if (file == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct")
+
+ file->s3r_handle = handle;
+ HDmemcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t));
+
+#if ROS3_STATS
+ if (FAIL == ros3_reset_stats(file))
+ HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics")
+#endif /* ROS3_STATS */
+
+ ret_value = (H5FD_t*)file;
+
+done:
+ if (ret_value == NULL) {
+ if (handle != NULL) {
+ if (FAIL == H5FD_s3comms_s3r_close(handle))
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle")
+ }
+ if (file != NULL) {
+ file = H5FL_FREE(H5FD_ros3_t, file);
+ }
+ curl_global_cleanup(); /* early cleanup because open failed */
+ } /* end if null return value (error) */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_ros3_open() */
+
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_close()
+ *
+ * Purpose:
+ *
+ * Close an HDF5 file.
+ *
+ * Return:
+ *
+ * SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_close() called.\n");
+#endif
+
+ /* Sanity checks
+ */
+ HDassert(file != NULL);
+ HDassert(file->s3r_handle != NULL);
+
+ /* Close the underlying request handle
+ */
+ if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close S3 request handle")
+ }
+
+#if ROS3_STATS
+ /* TODO: mechanism to re-target stats printout */
+ if (ros3_fprint_stats(stdout, file) == FAIL) {
+ HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL,
+ "problem while writing file statistics")
+ }
+#endif /* ROS3_STATS */
+
+ /* Release the file info
+ */
+ file = H5FL_FREE(H5FD_ros3_t, file);
+
+done:
+ curl_global_cleanup(); /* cleanup to answer init on open */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_close() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_cmp()
+ *
+ * Purpose:
+ *
+ * Compares two files belonging to this driver using an arbitrary
+ * (but consistent) ordering:
+ *
+ * + url scheme
+ * + url host
+ * + url port
+ * + url path
+ * + url query
+ * + fapl aws_region
+ * + fapl secret_id
+ * + fapl secret_key
+ *
+ * tl;dr -> check URL, check crentials
+ *
+ * Return:
+ *
+ * - Equivalent: 0
+ * - Not Equivalent: -1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-06
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5FD_ros3_cmp(
+ const H5FD_t *_f1,
+ const H5FD_t *_f2)
+{
+ const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1;
+ const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2;
+ const parsed_url_t *purl1 = NULL;
+ const parsed_url_t *purl2 = NULL;
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_cmp() called.\n");
+#endif
+
+ HDassert(f1->s3r_handle != NULL);
+ HDassert(f2->s3r_handle != NULL);
+
+ purl1 = (const parsed_url_t *)f1->s3r_handle->purl;
+ purl2 = (const parsed_url_t *)f2->s3r_handle->purl;
+ HDassert(purl1 != NULL);
+ HDassert(purl2 != NULL);
+ HDassert(purl1->scheme != NULL);
+ HDassert(purl2->scheme != NULL);
+ HDassert(purl1->host != NULL);
+ HDassert(purl2->host != NULL);
+
+ /* URL: SCHEME */
+ if (HDstrcmp(purl1->scheme, purl2->scheme)) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: HOST */
+ if (HDstrcmp(purl1->host, purl2->host)) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: PORT */
+ if (purl1->port && purl2->port) {
+ if (HDstrcmp(purl1->port, purl2->port)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (purl1->port) {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (purl2->port) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: PATH */
+ if (purl1->path && purl2->path) {
+ if (HDstrcmp(purl1->path, purl2->path)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (purl1->path && !purl2->path) {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (purl2->path && !purl1->path) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: QUERY */
+ if (purl1->query && purl2->query) {
+ if (HDstrcmp(purl1->query, purl2->query)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (purl1->query && !purl2->query) {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (purl2->query && !purl1->query) {
+ HGOTO_DONE(-1);
+ }
+
+ /* FAPL: AWS_REGION */
+ if (f1->fa.aws_region[0] != '\0' && f1->fa.aws_region[0] != '\0') {
+ if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (f1->fa.aws_region[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (f2->fa.aws_region[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+
+ /* FAPL: SECRET_ID */
+ if (f1->fa.secret_id[0] != '\0' && f1->fa.secret_id[0] != '\0') {
+ if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (f1->fa.secret_id[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (f2->fa.secret_id[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+
+ /* FAPL: SECRET_KEY */
+ if (f1->fa.secret_key[0] != '\0' && f1->fa.secret_key[0] != '\0') {
+ if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (f1->fa.secret_key[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (f2->fa.secret_key[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_ros3_cmp() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Note that since the ROS3 VFD is read only, most flags
+ * are irrelevant.
+ *
+ * The term "set" is highly misleading...
+ * stores/copies the supported flags in the out-pointer `flags`.
+ *
+ * Return: SUCCEED (Can't fail)
+ *
+ * Programmer: John Mainzer
+ * 9/11/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file,
+ unsigned long *flags /* out */)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_query() called.\n");
+#endif
+
+ /* Set the VFL feature flags that this driver supports */
+ if (flags) {
+ *flags = 0;
+ /* OK to perform data sieving for faster raw data reads & writes */
+ *flags |= H5FD_FEAT_DATA_SIEVE;
+ } /* end if */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* H5FD_ros3_query() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_get_eoa()
+ *
+ * Purpose:
+ *
+ * Gets the end-of-address marker for the file. The EOA marker
+ * is the first address past the last byte allocated in the
+ * format address space.
+ *
+ * Return:
+ *
+ * The end-of-address marker.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static haddr_t
+H5FD_ros3_get_eoa(const H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_get_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(file->eoa)
+
+} /* end H5FD_ros3_get_eoa() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_set_eoa()
+ *
+ * Purpose:
+ *
+ * Set the end-of-address marker for the file.
+ *
+ * Return:
+ *
+ * SUCCEED (can't fail)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_set_eoa(H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type,
+ haddr_t addr)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_set_eoa() called.\n");
+#endif
+
+ file->eoa = addr;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* H5FD_ros3_set_eoa() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_get_eof()
+ *
+ * Purpose:
+ *
+ * Returns the end-of-file marker.
+ *
+ * Return:
+ *
+ * EOF: the first address past the end of the "file", either the
+ * filesystem file or the HDF5 file.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static haddr_t
+H5FD_ros3_get_eof(const H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_get_eof() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(H5FD_s3comms_s3r_get_filesize(file->s3r_handle))
+
+} /* end H5FD_ros3_get_eof() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_get_handle()
+ *
+ * Purpose:
+ *
+ * Returns the S3 Request handle (s3r_t) of ros3 file driver.
+ *
+ * Returns:
+ *
+ * SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_get_handle(H5FD_t *_file,
+ hid_t H5_ATTR_UNUSED fapl,
+ void **file_handle)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_get_handle() called.\n");
+#endif
+
+ if (!file_handle) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid")
+ }
+
+ *file_handle = file->s3r_handle;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_get_handle() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_read()
+ *
+ * Purpose
+ *
+ * Reads SIZE bytes of data from FILE beginning at address ADDR
+ * into buffer BUF according to data transfer properties in DXPL_ID.
+ *
+ * Return:
+ *
+ * Success: `SUCCEED`
+ * - Result is stored in caller-supplied buffer BUF.
+ * Failure: `FAIL`
+ * - Unable to complete read.
+ * - Contents of buffer `buf` are undefined.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-??
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_read(H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type,
+ hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t addr, /* start offset */
+ size_t size, /* length of read */
+ void *buf) /* out */
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+ size_t filesize = 0;
+ herr_t ret_value = SUCCEED;
+#if ROS3_STATS
+ /* working variables for storing stats */
+ ros3_statsbin *bin = NULL;
+ unsigned bin_i = 0;
+#endif /* ROS3_STATS */
+
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_read() called.\n");
+#endif
+
+ HDassert(file != NULL);
+ HDassert(file->s3r_handle != NULL);
+ HDassert(buf != NULL);
+
+ filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
+
+ if ((addr > filesize) || ((addr + size) > filesize)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address")
+ }
+
+ if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) {
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read")
+ }
+
+#if ROS3_STATS
+
+ /* Find which "bin" this read fits in. Can be "overflow" bin.
+ */
+ for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) {
+ if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) {
+ break;
+ }
+ }
+ bin = (type == H5FD_MEM_DRAW)
+ ? &file->raw[bin_i]
+ : &file->meta[bin_i];
+
+ /* Store collected stats in appropriate bin
+ */
+ if (bin->count == 0) {
+ bin->min = size;
+ bin->max = size;
+ }
+ else {
+ if (size < bin->min) {
+ bin->min = size;
+ }
+ if (size > bin->max) {
+ bin->max = size;
+ }
+ }
+ bin->count++;
+ bin->bytes += (unsigned long long)size;
+
+#endif /* ROS3_STATS */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_read() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_write()
+ *
+ * Purpose:
+ *
+ * Write bytes to file.
+ * UNSUPPORTED IN READ-ONLY ROS3 VFD.
+ *
+ * Return:
+ *
+ * FAIL (Not possible with Read-Only S3 file.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type,
+ hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t H5_ATTR_UNUSED addr,
+ size_t H5_ATTR_UNUSED size,
+ const void H5_ATTR_UNUSED *buf)
+{
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_write() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL,
+ "cannot write to read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_ros3_write() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_truncate()
+ *
+ * Purpose:
+ *
+ * Makes sure that the true file size is the same (or larger)
+ * than the end-of-address.
+ *
+ * NOT POSSIBLE ON READ-ONLY S3 FILES.
+ *
+ * Return:
+ *
+ * FAIL (Not possible on Read-Only S3 files.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file,
+ hid_t H5_ATTR_UNUSED dxpl_id,
+ hbool_t H5_ATTR_UNUSED closing)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_truncate() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL,
+ "cannot truncate read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_truncate() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_lock()
+ *
+ * Purpose:
+ *
+ * Place an advisory lock on a file.
+ * No effect on Read-Only S3 file.
+ *
+ * Suggestion: remove lock/unlock from class
+ * > would result in error at H5FD_[un]lock() (H5FD.c)
+ *
+ * Return:
+ *
+ * SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file,
+ hbool_t H5_ATTR_UNUSED rw)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* end H5FD_ros3_lock() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_unlock()
+ *
+ * Purpose:
+ *
+ * Remove the existing lock on the file.
+ * No effect on Read-Only S3 file.
+ *
+ * Return:
+ *
+ * SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_unlock(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* end H5FD_ros3_unlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_validate_config()
+ *
+ * Purpose: Test to see if the supplied instance of H5FD_ros3_fapl_t
+ * contains internally consistant data. Return SUCCEED if so,
+ * and FAIL otherwise.
+ *
+ * Note the difference between internally consistant and
+ * correct. As we will have to try to access the target
+ * object to determine whether the supplied data is correct,
+ * we will settle for internal consistancy at this point
+ *
+ * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally
+ * consistant data, FAIL otherwise.
+ *
+ * Programmer: Jacob Smith
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HDassert(fa != NULL);
+
+ if ( fa->version != H5FD_CURR_ROS3_FAPL_T_VERSION ) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Unknown H5FD_ros3_fapl_t version");
+ }
+
+ /* if set to authenticate, region and id cannot be empty strings
+ */
+ if (fa->authenticate == TRUE) {
+ if ((fa->aws_region[0] == '\0') ||
+ (fa->secret_id[0] == '\0'))
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Inconsistent authentication information");
+ }
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_validate_config() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_fapl_ros3
+ *
+ * Purpose: Returns information about the ros3 file access property
+ * list though the function arguments.
+ *
+ * Return: Success: Non-negative
+ *
+ * Failure: Negative
+ *
+ * Programmer: John Mainzer
+ * 9/10/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_fapl_ros3(hid_t fapl_id,
+ H5FD_ros3_fapl_t *fa_out)
+{
+ const H5FD_ros3_fapl_t *fa = NULL;
+ H5P_genplist_t *plist = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa_out);
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5Pget_fapl_ros3() called.\n");
+#endif
+
+ if (fa_out == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL")
+ }
+
+ plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+ if (plist == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list")
+ }
+
+ if (H5FD_ROS3 != H5P_get_driver(plist)) {
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
+ }
+
+ fa = (const H5FD_ros3_fapl_t *)H5P_get_driver_info(plist);
+ if (fa == NULL) {
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info")
+ }
+
+ /* Copy the ros3 fapl data out */
+ HDmemcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t));
+
+done:
+ FUNC_LEAVE_API(ret_value)
+
+} /* end H5Pget_fapl_ros3() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_fapl_get
+ *
+ * Purpose: Gets a file access property list which could be used to
+ * create an identical file.
+ *
+ * Return: Success: Ptr to new file access property list value.
+ *
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_ros3_fapl_get(H5FD_t *_file)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t*)_file;
+ H5FD_ros3_fapl_t *fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ fa = (H5FD_ros3_fapl_t *)H5MM_calloc(sizeof(H5FD_ros3_fapl_t));
+ if (fa == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
+ "memory allocation failed")
+ }
+
+ /* Copy the fields of the structure */
+ HDmemcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t));
+
+ /* Set return value */
+ ret_value = fa;
+
+done:
+ if (ret_value == NULL) {
+ if (fa != NULL) {
+ H5MM_xfree(fa);
+ }
+ }
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_fapl_get() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_fapl_copy
+ *
+ * Purpose: Copies the ros3-specific file access properties.
+ *
+ * Return: Success: Ptr to a new property list
+ *
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_ros3_fapl_copy(const void *_old_fa)
+{
+ const H5FD_ros3_fapl_t *old_fa = (const H5FD_ros3_fapl_t*)_old_fa;
+ H5FD_ros3_fapl_t *new_fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ new_fa = (H5FD_ros3_fapl_t *)H5MM_malloc(sizeof(H5FD_ros3_fapl_t));
+ if (new_fa == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
+ "memory allocation failed");
+ }
+
+ HDmemcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t));
+ ret_value = new_fa;
+
+done:
+ if (ret_value == NULL) {
+ if (new_fa != NULL) {
+ H5MM_xfree(new_fa);
+ }
+ }
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_fapl_copy() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_fapl_free
+ *
+ * Purpose: Frees the ros3-specific file access properties.
+ *
+ * Return: SUCCEED (cannot fail)
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_fapl_free(void *_fa)
+{
+ H5FD_ros3_fapl_t *fa = (H5FD_ros3_fapl_t*)_fa;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(fa != NULL); /* sanity check */
+
+ H5MM_xfree(fa);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* end H5FD_ros3_fapl_free() */
+
+#if ROS3_STATS
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: ros3_reset_stats()
+ *
+ * Purpose:
+ *
+ * Reset the stats collection elements in this virtual file structure.
+ *
+ * Clears any set data in stats bins; initializes/zeroes values.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - Occurs if the file is invalid somehow
+ *
+ * Programmer: Jacob Smith
+ * 2017-12-08
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+ros3_reset_stats(H5FD_ros3_t *file)
+{
+ unsigned i = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDprintf("ros3_reset_stats() called\n");
+#endif
+
+ if (file == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "file was null");
+ }
+
+ for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) {
+ file->raw[i].bytes = 0;
+ file->raw[i].count = 0;
+ file->raw[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ file->raw[i].max = 0;
+
+ file->meta[i].bytes = 0;
+ file->meta[i].count = 0;
+ file->meta[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ file->meta[i].max = 0;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* end ros3_reset_stats() */
+
+#endif /* ROS3_STATS */
+#if ROS3_STATS
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: ros3_fprint_stats()
+ *
+ * Purpose:
+ *
+ * Tabulate and pretty-print statistics for this virtual file.
+ *
+ * Should be called upon file close.
+ *
+ * Shows number of reads and bytes read, broken down by
+ * "raw" (H5FD_MEM_DRAW)
+ * or "meta" (any other flag)
+ *
+ * Prints filename and listing of total number of reads and bytes read,
+ * both as a grand total and separate meta- and rawdata reads.
+ *
+ * If any reads were done, prints out two tables:
+ *
+ * 1. overview of raw- and metadata reads
+ * - min (smallest size read)
+ * - average of size read
+ * - k,M,G suffixes by powers of 1024 (2^10)
+ * - max (largest size read)
+ * 2. tabulation of "bins", sepraring reads into exponentially-larger
+ * ranges of size.
+ * - columns for number of reads, total bytes, and average size, with
+ * separate sub-colums for raw- and metadata reads.
+ * - each row represents one bin, identified by the top of its range
+ *
+ * Bin ranges can be modified with pound-defines at the top of this file.
+ *
+ * Bins without any reads in their bounds are not printed.
+ *
+ * An "overflow" bin is also present, to catch "big" reads.
+ *
+ * Output for all bins (and range ceiling and average size report)
+ * is divied by powers of 1024. By corollary, four digits before the decimal
+ * is valid.
+ *
+ * - 41080 bytes is represented by 40.177k, not 41.080k
+ * - 1004.831M represents approx. 1052642000 bytes
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - occurs if the file passed in is invalid
+ * - TODO: if stream is invalid? how can we check this?
+ *
+ * Programmer: Jacob Smith
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+ros3_fprint_stats(FILE *stream,
+ const H5FD_ros3_t *file)
+{
+ herr_t ret_value = SUCCEED;
+ parsed_url_t *purl = NULL;
+ unsigned i = 0;
+ unsigned long count_meta = 0;
+ unsigned long count_raw = 0;
+ double average_meta = 0.0;
+ double average_raw = 0.0;
+ unsigned long long min_meta = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ unsigned long long min_raw = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ unsigned long long max_meta = 0;
+ unsigned long long max_raw = 0;
+ unsigned long long bytes_raw = 0;
+ unsigned long long bytes_meta = 0;
+ double re_dub = 0.0; /* re-usable double variable */
+ unsigned suffix_i = 0;
+ const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' };
+
+
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ if (stream == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "file stream cannot be null" );
+ }
+ if (file == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "file cannot be null");
+ }
+ if (file->s3r_handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "s3 request handle cannot be null");
+ }
+ if (file->s3r_handle->purl == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "parsed url structure cannot be null");
+ }
+ purl = file->s3r_handle->purl;
+
+ /******************
+ * PRINT FILENAME *
+ ******************/
+
+ HDfprintf(stream, "stats for %s://%s", purl->scheme, purl->host);
+ if (purl->port != NULL && purl->port[0] != '\0')
+ HDfprintf(stream, ":%s", purl->port);
+ if (purl->query != NULL && purl->query[0] != '\0') {
+ if (purl->path != NULL && purl->path[0] != '\0')
+ HDfprintf(stream, "/%s", purl->path);
+ else
+ HDfprintf(stream, "/");
+ HDfprintf(stream, "?%s", purl->query);
+ } else if (purl->path != NULL && purl->path[0] != '\0') {
+ HDfprintf(stream, "/%s", purl->path);
+ }
+ HDfprintf(stream, "\n");
+
+ /*******************
+ * AGGREGATE STATS *
+ *******************/
+
+ for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) {
+ const ros3_statsbin *r = &file->raw[i];
+ const ros3_statsbin *m = &file->meta[i];
+
+ if (m->min < min_meta) min_meta = m->min;
+ if (r->min < min_raw) min_raw = r->min;
+ if (m->max > max_meta) max_meta = m->max;
+ if (r->max > max_raw) max_raw = r->max;
+
+ count_raw += r->count;
+ count_meta += m->count;
+ bytes_raw += r->bytes;
+ bytes_meta += m->bytes;
+ }
+ if (count_raw > 0)
+ average_raw = (double)bytes_raw / (double)count_raw;
+ if (count_meta > 0)
+ average_meta = (double)bytes_meta / (double)count_meta;
+
+ /******************
+ * PRINT OVERVIEW *
+ ******************/
+
+ HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n",
+ count_raw + count_meta, count_meta, count_raw);
+ HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n",
+ bytes_raw + bytes_meta, bytes_meta, bytes_raw);
+
+ if (count_raw + count_meta == 0)
+ goto done;
+
+ /*************************
+ * PRINT AGGREGATE STATS *
+ *************************/
+
+ HDfprintf(stream, "SIZES meta raw\n");
+ HDfprintf(stream, " min ");
+ if (count_meta == 0) {
+ HDfprintf(stream, " 0.000 ");
+ } else {
+ re_dub = (double)min_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+ }
+
+ if (count_raw == 0) {
+ HDfprintf(stream, " 0.000 \n");
+ } else {
+ re_dub = (double)min_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+ }
+
+ HDfprintf(stream, " avg ");
+ re_dub = (double)average_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)average_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ HDfprintf(stream, " max ");
+ re_dub = (double)max_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)max_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ /******************************
+ * PRINT INDIVIDUAL BIN STATS *
+ ******************************/
+
+ HDfprintf(stream,
+ "BINS # of reads total bytes average size\n");
+ HDfprintf(stream,
+ " up-to meta raw meta raw meta raw\n");
+
+ for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) {
+ const ros3_statsbin *m;
+ const ros3_statsbin *r;
+ unsigned long long range_end = 0;
+ char bm_suffix = ' '; /* bytes-meta */
+ double bm_val = 0.0;
+ char br_suffix = ' '; /* bytes-raw */
+ double br_val = 0.0;
+ char am_suffix = ' '; /* average-meta */
+ double am_val = 0.0;
+ char ar_suffix = ' '; /* average-raw */
+ double ar_val = 0.0;
+
+ m = &file->meta[i];
+ r = &file->raw[i];
+ if (r->count == 0 && m->count == 0)
+ continue;
+
+ range_end = ros3_stats_boundaries[i];
+
+ if (i == ROS3_STATS_BIN_COUNT) {
+ range_end = ros3_stats_boundaries[i-1];
+ HDfprintf(stream, ">");
+ } else {
+ HDfprintf(stream, " ");
+ }
+
+ bm_val = (double)m->bytes;
+ for (suffix_i = 0; bm_val >= 1024.0; suffix_i++)
+ bm_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ bm_suffix = suffixes[suffix_i];
+
+ br_val = (double)r->bytes;
+ for (suffix_i = 0; br_val >= 1024.0; suffix_i++)
+ br_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ br_suffix = suffixes[suffix_i];
+
+ if (m->count > 0)
+ am_val = (double)(m->bytes) / (double)(m->count);
+ for (suffix_i = 0; am_val >= 1024.0; suffix_i++)
+ am_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ am_suffix = suffixes[suffix_i];
+
+ if (r->count > 0)
+ ar_val = (double)(r->bytes) / (double)(r->count);
+ for (suffix_i = 0; ar_val >= 1024.0; suffix_i++)
+ ar_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ ar_suffix = suffixes[suffix_i];
+
+ re_dub = (double)range_end;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+
+ HDfprintf(stream,
+ " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n",
+ re_dub, suffixes[suffix_i], /* bin ceiling */
+ m->count, /* metadata reads */
+ r->count, /* rawdata reads */
+ bm_val, bm_suffix, /* metadata bytes */
+ br_val, br_suffix, /* rawdata bytes */
+ am_val, am_suffix, /* metadata average */
+ ar_val, ar_suffix); /* rawdata average */
+
+ fflush(stream);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* ros3_fprint_stats */
+#endif /* ROS3_STATS */
+
+#endif /* H5_HAVE_ROS3_VFD */
+
diff --git a/src/H5FDros3.h b/src/H5FDros3.h
new file mode 100644
index 0000000..8ecd7f0
--- /dev/null
+++ b/src/H5FDros3.h
@@ -0,0 +1,106 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Programmer: John Mainzer
+ * 2017-10-10
+ *
+ * Purpose: The public header file for the ros3 driver.
+ */
+#ifndef H5FDros3_H
+#define H5FDros3_H
+
+#ifdef H5_HAVE_ROS3_VFD
+#define H5FD_ROS3 (H5FD_ros3_init())
+#else
+#define H5FD_ROS3 (H5I_INVALID_HID)
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/****************************************************************************
+ *
+ * Structure: H5FD_ros3_fapl_t
+ *
+ * Purpose:
+ *
+ * H5FD_ros3_fapl_t is a public structure that is used to pass S3
+ * authentication data to the appropriate S3 VFD via the FAPL. A pointer
+ * to an instance of this structure is a parameter to H5Pset_fapl_ros3()
+ * and H5Pget_fapl_ros3().
+ *
+ *
+ *
+ * `version` (int32_t)
+ *
+ * Version number of the H5FD_ros3_fapl_t structure. Any instance passed
+ * to the above calls must have a recognized version number, or an error
+ * will be flagged.
+ *
+ * This field should be set to H5FD_CURR_ROS3_FAPL_T_VERSION.
+ *
+ * `authenticate` (hbool_t)
+ *
+ * Flag TRUE or FALSE whether or not requests are to be authenticated
+ * with the AWS4 algorithm.
+ * If TRUE, `aws_region`, `secret_id`, and `secret_key` must be populated.
+ * If FALSE, those three components are unused.
+ *
+ * `aws_region` (char[])
+ *
+ * String: name of the AWS "region" of the host, e.g. "us-east-1".
+ *
+ * `secret_id` (char[])
+ *
+ * String: "Access ID" for the resource.
+ *
+ * `secret_key` (char[])
+ *
+ * String: "Secret Access Key" associated with the ID and resource.
+ *
+ ****************************************************************************/
+
+#define H5FD_CURR_ROS3_FAPL_T_VERSION 1
+
+#define H5FD_ROS3_MAX_REGION_LEN 32
+#define H5FD_ROS3_MAX_SECRET_ID_LEN 128
+#define H5FD_ROS3_MAX_SECRET_KEY_LEN 128
+
+typedef struct H5FD_ros3_fapl_t {
+ int32_t version;
+ hbool_t authenticate;
+ char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1];
+ char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1];
+ char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1];
+} H5FD_ros3_fapl_t;
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+H5_DLL hid_t H5FD_ros3_init(void);
+H5_DLL void H5FD_ros3_term(void);
+H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out);
+H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+#endif /* ifndef H5FDros3_H */
+
+
diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c
new file mode 100644
index 0000000..f08e9d5
--- /dev/null
+++ b/src/H5FDs3comms.c
@@ -0,0 +1,3593 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*****************************************************************************
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Source for S3 Communications module
+ *
+ * ***NOT A FILE DRIVER***
+ *
+ * Provide functions and structures required for interfacing with Amazon
+ * Simple Storage Service (S3).
+ *
+ * Provide S3 object access as if it were a local file.
+ *
+ * Connect to remote host, send and receive HTTP requests and responses
+ * as part of the AWS REST API, authenticating requests as appropriate.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-30
+ *
+ *****************************************************************************/
+
+/****************/
+/* Module Setup */
+/****************/
+
+/***********/
+/* Headers */
+/***********/
+
+#include "H5private.h" /* generic functions */
+#include "H5Eprivate.h" /* error handling */
+#include "H5MMprivate.h" /* memory management */
+#include "H5FDs3comms.h" /* S3 Communications */
+
+/****************/
+/* Local Macros */
+/****************/
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/* toggle debugging (enable with 1)
+ */
+#define S3COMMS_DEBUG 0
+
+/* manipulate verbosity of CURL output
+ * operates separately from S3COMMS_DEBUG
+ *
+ * 0 -> no explicit curl output
+ * 1 -> on error, print failure info to stderr
+ * 2 -> in addition to above, print information for all performs; sets all
+ * curl handles with CURLOPT_VERBOSE
+ */
+#define S3COMMS_CURL_VERBOSITY 0
+
+/* size to allocate for "bytes=<first_byte>[-<last_byte>]" HTTP Range value
+ */
+#define S3COMMS_MAX_RANGE_STRING_SIZE 128
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/********************/
+/* Local Structures */
+/********************/
+
+/* struct s3r_datastruct
+ * Structure passed to curl write callback
+ * pointer to data region and record of bytes written (offset)
+ */
+struct s3r_datastruct {
+ unsigned long magic;
+ char *data;
+ size_t size;
+};
+#define S3COMMS_CALLBACK_DATASTRUCT_MAGIC 0x28c2b2ul
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+size_t curlwritecallback(char *ptr,
+ size_t size,
+ size_t nmemb,
+ void *userdata);
+
+herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle);
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/*************/
+/* Functions */
+/*************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: curlwritecallback()
+ *
+ * Purpose:
+ *
+ * Function called by CURL to write received data.
+ *
+ * Writes bytes to `userdata`.
+ *
+ * Internally manages number of bytes processed.
+ *
+ * Return:
+ *
+ * - Number of bytes processed.
+ * - Should equal number of bytes passed to callback.
+ * - Failure will result in curl error: CURLE_WRITE_ERROR.
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-17
+ *
+ *----------------------------------------------------------------------------
+ */
+size_t
+curlwritecallback(char *ptr,
+ size_t size,
+ size_t nmemb,
+ void *userdata)
+{
+ struct s3r_datastruct *sds = (struct s3r_datastruct *)userdata;
+ size_t product = (size * nmemb);
+ size_t written = 0;
+
+ if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) {
+ return written;
+ }
+
+ if (size > 0) {
+ HDmemcpy(&(sds->data[sds->size]), ptr, product);
+ sds->size += product;
+ written = product;
+ }
+
+ return written;
+
+} /* end curlwritecallback() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_hrb_node_set()
+ *
+ * Purpose:
+ *
+ * Create, insert, modify, and remove elements in a field node list.
+ *
+ * `name` cannot be null; will return FAIL and list will be unaltered.
+ *
+ * Entries are accessed via the lowercase representation of their name:
+ * "Host", "host", and "hOSt" would all access the same node,
+ * but name's case is relevant in HTTP request output.
+ *
+ * List pointer `L` must always point to either of :
+ * - header node with lowest alphabetical order (by lowername)
+ * - NULL, if list is empty
+ *
+ * Types of operations:
+ *
+ * - CREATE
+ * - If `L` is NULL and `name` and `value` are not NULL,
+ * a new node is created at `L`, starting a list.
+ * - MODIFY
+ * - If a node is found with a matching lowercase name and `value`
+ * is not NULL, the existing name, value, and cat values are released
+ * and replaced with the new data.
+ * - No modifications are made to the list pointers.
+ * - REMOVE
+ * - If `value` is NULL, will attempt to remove node with matching
+ * lowercase name.
+ * - If no match found, returns FAIL and list is not modified.
+ * - When removing a node, all its resources is released.
+ * - If removing the last node in the list, list pointer is set to NULL.
+ * - INSERT
+ * - If no nodes exists with matching lowercase name and `value`
+ * is not NULL, a new node is created, inserted into list
+ * alphabetically by lowercase name.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - List was successfully modified
+ * - FAILURE: `FAIL`
+ * - Unable to perform operation
+ * - Forbidden (attempting to remove absent node, e.g.)
+ * - Internal error
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-22
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_hrb_node_set(
+ hrb_node_t **L,
+ const char *name,
+ const char *value)
+{
+ size_t i = 0;
+ char *valuecpy = NULL;
+ char *namecpy = NULL;
+ size_t namelen = 0;
+ char *lowername = NULL;
+ char *nvcat = NULL;
+ hrb_node_t *node_ptr = NULL;
+ hrb_node_t *new_node = NULL;
+ hbool_t is_looking = TRUE;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_hrb_node_set.\n");
+ HDprintf("NAME: %s\n", name);
+ HDprintf("VALUE: %s\n", value);
+ HDprintf("LIST:\n->");
+ for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) {
+ HDfprintf(stdout, "{%s}\n->", node_ptr->cat);
+ }
+ HDprintf("(null)\n");
+ fflush(stdout);
+ node_ptr = NULL;
+#endif
+
+ if (name == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to operate on null name.\n");
+ }
+ namelen = HDstrlen(name);
+
+ /***********************
+ * PREPARE ALL STRINGS *
+ **********************/
+
+ /* copy and lowercase name
+ */
+ lowername = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
+ if (lowername == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for lowercase name copy.\n");
+ }
+ for (i = 0; i < namelen; i++) {
+ lowername[i] = (char)tolower((int)name[i]);
+ }
+ lowername[namelen] = 0;
+
+ /* If value supplied, copy name, value, and concatenated "name: value".
+ * If NULL, we will be removing a node or doing nothing, so no need for
+ * copies
+ */
+ if (value != NULL) {
+ int ret = 0;
+ size_t valuelen = HDstrlen(value);
+ size_t catlen = namelen + valuelen + 2; /* +2 from ": " */
+ size_t catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */
+
+
+ namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
+ if (namecpy == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for name copy.\n");
+ }
+ HDmemcpy(namecpy, name, (namelen + 1));
+
+ valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1));
+ if (valuecpy == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for value copy.\n");
+ }
+ HDmemcpy(valuecpy, value, (valuelen + 1));
+
+ nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite);
+ if (nvcat == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for concatenated string.\n");
+ }
+ ret = HDsnprintf(nvcat, catwrite, "%s: %s", name, value);
+ if (ret < 0 || (size_t)ret > catlen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot concatenate `%s: %s", name, value);
+ }
+ HDassert( catlen == HDstrlen(nvcat) );
+
+ /* create new_node, should we need it
+ */
+ new_node = (hrb_node_t *)H5MM_malloc(sizeof(hrb_node_t));
+ if (new_node == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for new set.\n");
+ }
+
+ new_node->magic = S3COMMS_HRB_NODE_MAGIC;
+ new_node->name = NULL;
+ new_node->value = NULL;
+ new_node->cat = NULL;
+ new_node->lowername = NULL;
+ new_node->next = NULL;
+ }
+
+ /***************
+ * ACT ON LIST *
+ ***************/
+
+ if (*L == NULL) {
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove node from empty list");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("CREATE NEW\n"); fflush(stdout);
+#endif
+ /*******************
+ * CREATE NEW LIST *
+ *******************/
+
+ new_node->cat = nvcat;
+ new_node->name = namecpy;
+ new_node->lowername = lowername;
+ new_node->value = valuecpy;
+
+ *L = new_node;
+ goto done; /* bypass further seeking */
+ }
+ }
+
+ /* sanity-check pointer passed in
+ */
+ HDassert( (*L) != NULL );
+ HDassert( (*L)->magic == S3COMMS_HRB_NODE_MAGIC );
+ node_ptr = (*L);
+
+ /* Check whether to modify/remove first node in list
+ */
+ if (strcmp(lowername, node_ptr->lowername) == 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+#if S3COMMS_DEBUG
+HDprintf("REMOVE HEAD\n"); fflush(stdout);
+#endif
+ /***************
+ * REMOVE HEAD *
+ ***************/
+
+ *L = node_ptr->next;
+
+#if S3COMMS_DEBUG
+HDprintf("FREEING CAT (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->cat);
+#if S3COMMS_DEBUG
+HDprintf("FREEING LOWERNAME (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->lowername);
+#if S3COMMS_DEBUG
+HDprintf("FREEING NAME (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->name);
+#if S3COMMS_DEBUG
+HDprintf("FREEING VALUE (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->value);
+#if S3COMMS_DEBUG
+HDprintf("MAGIC OK? %s\n",
+ (node_ptr->magic == S3COMMS_HRB_NODE_MAGIC) ? "YES" : "NO");
+fflush(stdout);
+#endif
+ HDassert( node_ptr->magic == S3COMMS_HRB_NODE_MAGIC );
+ node_ptr->magic += 1ul;
+#if S3COMMS_DEBUG
+HDprintf("FREEING POINTER\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr);
+
+#if S3COMMS_DEBUG
+HDprintf("FREEING WORKING LOWERNAME\n"); fflush(stdout);
+#endif
+ H5MM_xfree(lowername); lowername = NULL;
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("MODIFY HEAD\n"); fflush(stdout);
+#endif
+ /***************
+ * MODIFY HEAD *
+ ***************/
+
+ H5MM_xfree(node_ptr->cat);
+ H5MM_xfree(node_ptr->name);
+ H5MM_xfree(node_ptr->value);
+
+ node_ptr->name = namecpy;
+ node_ptr->value = valuecpy;
+ node_ptr->cat = nvcat;
+
+ H5MM_xfree(lowername);
+ lowername = NULL;
+ new_node->magic += 1ul;
+ H5MM_xfree(new_node);
+ new_node = NULL;
+ }
+ }
+ else
+ if (strcmp(lowername, node_ptr->lowername) < 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove a node 'before' head");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("PREPEND NEW HEAD\n"); fflush(stdout);
+#endif
+ /*******************
+ * INSERT NEW HEAD *
+ *******************/
+
+ new_node->name = namecpy;
+ new_node->value = valuecpy;
+ new_node->lowername = lowername;
+ new_node->cat = nvcat;
+ new_node->next = node_ptr;
+ *L = new_node;
+ }
+ }
+
+ /***************
+ * SEARCH LIST *
+ ***************/
+
+ while (is_looking) {
+ if (node_ptr->next == NULL) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove absent node");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("APPEND A NODE\n"); fflush(stdout);
+#endif
+ /*******************
+ * APPEND NEW NODE *
+ *******************/
+
+ HDassert( strcmp(lowername, node_ptr->lowername) > 0 );
+ new_node->name = namecpy;
+ new_node->value = valuecpy;
+ new_node->lowername = lowername;
+ new_node->cat = nvcat;
+ node_ptr->next = new_node;
+ }
+ }
+ else
+ if (strcmp(lowername, node_ptr->next->lowername) < 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove absent node");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("INSERT A NODE\n"); fflush(stdout);
+#endif
+ /*******************
+ * INSERT NEW NODE *
+ *******************/
+
+ HDassert( strcmp(lowername, node_ptr->lowername) > 0 );
+ new_node->name = namecpy;
+ new_node->value = valuecpy;
+ new_node->lowername = lowername;
+ new_node->cat = nvcat;
+ new_node->next = node_ptr->next;
+ node_ptr->next = new_node;
+ }
+ }
+ else
+ if (strcmp(lowername, node_ptr->next->lowername) == 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ /*****************
+ * REMOVE A NODE *
+ *****************/
+
+ hrb_node_t *tmp = node_ptr->next;
+ node_ptr->next = tmp->next;
+
+#if S3COMMS_DEBUG
+HDprintf("REMOVE A NODE\n"); fflush(stdout);
+#endif
+ H5MM_xfree(tmp->cat);
+ H5MM_xfree(tmp->lowername);
+ H5MM_xfree(tmp->name);
+ H5MM_xfree(tmp->value);
+
+ HDassert( tmp->magic == S3COMMS_HRB_NODE_MAGIC );
+ tmp->magic += 1ul;
+ H5MM_xfree(tmp);
+
+ H5MM_xfree(lowername);
+ lowername = NULL;
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("MODIFY A NODE\n"); fflush(stdout);
+#endif
+ /*****************
+ * MODIFY A NODE *
+ *****************/
+
+ node_ptr = node_ptr->next;
+ H5MM_xfree(node_ptr->name);
+ H5MM_xfree(node_ptr->value);
+ H5MM_xfree(node_ptr->cat);
+
+ HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC );
+ new_node->magic += 1ul;
+ H5MM_xfree(new_node);
+ H5MM_xfree(lowername);
+ new_node = NULL;
+ lowername = NULL;
+
+ node_ptr->name = namecpy;
+ node_ptr->value = valuecpy;
+ node_ptr->cat = nvcat;
+ }
+ }
+ else {
+ /****************
+ * KEEP LOOKING *
+ ****************/
+
+ node_ptr = node_ptr->next;
+ }
+ } /* end while is_looking */
+
+done:
+ if (ret_value == FAIL) {
+ /* clean up
+ */
+ if (nvcat != NULL) { H5MM_xfree(nvcat); }
+ if (namecpy != NULL) { H5MM_xfree(namecpy); }
+ if (lowername != NULL) { H5MM_xfree(lowername); }
+ if (valuecpy != NULL) { H5MM_xfree(valuecpy); }
+ if (new_node != NULL) {
+ HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC );
+ new_node->magic += 1ul;
+ H5MM_xfree(new_node);
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_hrb_node_set() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_hrb_destroy()
+ *
+ * Purpose:
+ *
+ * Destroy and free resources _directly_ associated with an HTTP Buffer.
+ *
+ * Takes a pointer to pointer to the buffer structure.
+ * This allows for the pointer itself to be NULLed from within the call.
+ *
+ * If buffer or buffer pointer is NULL, there is no effect.
+ *
+ * Headers list at `first_header` is not touched.
+ *
+ * - Programmer should re-use or destroy `first_header` pointer
+ * (hrb_node_t *) as suits their purposes.
+ * - Recommend fetching prior to destroy()
+ * e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);`
+ * or maintaining an external reference.
+ * - Destroy node/list separately as appropriate
+ * - Failure to account for this will result in a memory leak.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - successfully released buffer resources
+ * - if `buf` is NULL or `*buf` is NULL, no effect
+ * - FAILURE: `FAIL`
+ * - `buf->magic != S3COMMS_HRB_MAGIC`
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-21
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_hrb_destroy(hrb_t **_buf)
+{
+ hrb_t *buf = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_hrb_destroy.\n");
+#endif
+
+ if (_buf != NULL && *_buf != NULL) {
+ buf = *_buf;
+ if (buf->magic != S3COMMS_HRB_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "pointer's magic does not match.\n");
+ }
+
+ H5MM_xfree(buf->verb);
+ H5MM_xfree(buf->version);
+ H5MM_xfree(buf->resource);
+ buf->magic += 1ul;
+ H5MM_xfree(buf);
+ *_buf = NULL;
+ } /* end if `_buf` has some value */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_s3comms_hrb_destroy() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_hrb_init_request()
+ *
+ * Purpose:
+ *
+ * Create a new HTTP Request Buffer
+ *
+ * All non-null arguments should be null-terminated strings.
+ *
+ * If `verb` is NULL, defaults to "GET".
+ * If `http_version` is NULL, defaults to "HTTP/1.1".
+ *
+ * `resource` cannot be NULL; should be string beginning with slash
+ * character ('/').
+ *
+ * All strings are copied into the structure, making them safe from
+ * modification in source strings.
+ *
+ * Return:
+ *
+ * - SUCCESS: pointer to new `hrb_t`
+ * - FAILURE: `NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-21
+ *
+ *----------------------------------------------------------------------------
+ */
+hrb_t *
+H5FD_s3comms_hrb_init_request(const char *_verb,
+ const char *_resource,
+ const char *_http_version)
+{
+ hrb_t *request = NULL;
+ char *res = NULL;
+ size_t reslen = 0;
+ hrb_t *ret_value = NULL;
+ char *verb = NULL;
+ size_t verblen = 0;
+ char *vrsn = NULL;
+ size_t vrsnlen = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_hrb_init_request.\n");
+#endif
+
+ if (_resource == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "resource string cannot be null.\n");
+ }
+
+ /* populate valid NULLs with defaults
+ */
+ if (_verb == NULL) {
+ _verb = "GET";
+ }
+
+ if (_http_version == NULL) {
+ _http_version = "HTTP/1.1";
+ }
+
+ /* malloc space for and prepare structure
+ */
+ request = (hrb_t *)H5MM_malloc(sizeof(hrb_t));
+ if (request == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "no space for request structure");
+ }
+ request->magic = S3COMMS_HRB_MAGIC;
+ request->body = NULL;
+ request->body_len = 0;
+ request->first_header = NULL;
+
+
+
+ /* malloc and copy strings for the structure
+ */
+ reslen = HDstrlen(_resource);
+
+ if (_resource[0] == '/') {
+ res = (char *)H5MM_malloc(sizeof(char) * (reslen+1));
+ if (res == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "no space for resource string");
+ }
+ HDmemcpy(res, _resource, (reslen+1));
+ }
+ else {
+ res = (char *)H5MM_malloc(sizeof(char) * (reslen+2));
+ if (res == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "no space for resource string");
+ }
+ *res = '/';
+ HDmemcpy((&res[1]), _resource, (reslen+1));
+ HDassert( (reslen+1) == HDstrlen(res) );
+ } /* end if (else resource string not starting with '/') */
+
+ verblen = HDstrlen(_verb) + 1;
+ verb = (char *)H5MM_malloc(sizeof(char) * verblen);
+ if (verb == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "no space for verb string");
+ }
+ HDstrncpy(verb, _verb, verblen);
+
+ vrsnlen = HDstrlen(_http_version) + 1;
+ vrsn = (char *)H5MM_malloc(sizeof(char) * vrsnlen);
+ if (vrsn == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "no space for http-version string");
+ }
+ HDstrncpy(vrsn, _http_version, vrsnlen);
+
+
+
+ /* place new copies into structure
+ */
+ request->resource = res;
+ request->verb = verb;
+ request->version = vrsn;
+
+ ret_value = request;
+
+done:
+ /* if there is an error, clean up after ourselves
+ */
+ if (ret_value == NULL) {
+ if (request != NULL) H5MM_xfree(request);
+ if (vrsn != NULL) H5MM_xfree(vrsn);
+ if (verb != NULL) H5MM_xfree(verb);
+ if (res != NULL) H5MM_xfree(res);
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_s3comms_hrb_init_request() */
+
+
+/****************************************************************************
+ * S3R FUNCTIONS
+ ****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_close()
+ *
+ * Purpose:
+ *
+ * Close communications through given S3 Request Handle (`s3r_t`)
+ * and clean up associated resources.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - fails if handle is null or has invalid magic number
+ *
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-31
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_s3r_close(s3r_t *handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_close.\n");
+#endif
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n");
+ }
+ if (handle->magic != S3COMMS_S3R_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n");
+ }
+
+ curl_easy_cleanup(handle->curlhandle);
+
+ H5MM_xfree(handle->secret_id);
+ H5MM_xfree(handle->region);
+ H5MM_xfree(handle->signing_key);
+
+ HDassert( handle->httpverb != NULL );
+ H5MM_xfree(handle->httpverb);
+
+ if (FAIL == H5FD_s3comms_free_purl(handle->purl)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to release parsed url structure")
+ }
+
+ H5MM_xfree(handle);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_s3comms_s3r_close */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_get_filesize()
+ *
+ * Purpose:
+ *
+ * Retrieve the filesize of an open request handle.
+ *
+ * Wrapper "getter" to hide implementation details.
+ *
+ *
+ * Return:
+ *
+ * - SUCCESS: size of file, in bytes, if handle is valid.
+ * - FAILURE: 0, if handle is NULL or undefined.
+ *
+ * Programmer: Jacob Smith 2017-01-14
+ *
+ *----------------------------------------------------------------------------
+ */
+size_t
+H5FD_s3comms_s3r_get_filesize(s3r_t *handle)
+{
+ size_t ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ if (handle != NULL) {
+ ret_value = handle->filesize;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_s3comms_s3r_get_filesize */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_getsize()
+ *
+ * Purpose:
+ *
+ * Get the number of bytes of handle's target resource.
+ *
+ * Sets handle and curlhandle with to enact an HTTP HEAD request on file,
+ * and parses received headers to extract "Content-Length" from response
+ * headers, storing file size at `handle->filesize`.
+ *
+ * Critical step in opening (initiating) an `s3r_t` handle.
+ *
+ * Wraps `s3r_read()`.
+ * Sets curlhandle to write headers to a temporary buffer (using extant
+ * write callback) and provides no buffer for body.
+ *
+ * Upon exit, unsets HTTP HEAD settings from curl handle, returning to
+ * initial state. In event of error, curl handle state is undefined and is
+ * not to be trusted.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-23
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_s3r_getsize(s3r_t *handle)
+{
+ uintmax_t content_length = 0;
+ CURL *curlh = NULL;
+ char *end = NULL;
+ char *headerresponse = NULL;
+ struct s3r_datastruct sds = {
+ S3COMMS_CALLBACK_DATASTRUCT_MAGIC,
+ NULL,
+ 0 };
+ char *start = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_getsize.\n");
+#endif
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n");
+ }
+ if (handle->magic != S3COMMS_S3R_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n");
+ }
+ if (handle->curlhandle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has bad (null) curlhandle.\n")
+ }
+
+ /********************
+ * PREPARE FOR HEAD *
+ ********************/
+
+ curlh = handle->curlhandle;
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_NOBODY,
+ 1L) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_NOBODY). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HEADERDATA,
+ &sds) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_HEADERDATA). "
+ "(placeholder flags)");
+ }
+
+ HDassert( handle->httpverb == NULL );
+ handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16);
+ if (handle->httpverb == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "unable to allocate space for S3 request HTTP verb");
+ }
+ HDmemcpy(handle->httpverb, "HEAD", 5);
+
+ headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER);
+ if (headerresponse == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "unable to allocate space for curl header response");
+ }
+ sds.data = headerresponse;
+
+ /*******************
+ * PERFORM REQUEST *
+ *******************/
+
+ /* these parameters fetch the entire file,
+ * but, with a NULL destination and NOBODY and HEADERDATA supplied above,
+ * only http metadata will be sent by server and recorded by s3comms
+ */
+ if (FAIL ==
+ H5FD_s3comms_s3r_read(handle, 0, 0, NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem in reading during getsize.\n");
+ }
+
+ if (sds.size > CURL_MAX_HTTP_HEADER) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "HTTP metadata buffer overrun\n");
+ } else if (sds.size == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "No HTTP metadata\n");
+#if S3COMMS_DEBUG
+ } else {
+ HDfprintf(stderr, "GETSIZE: OK\n");
+#endif
+ }
+
+
+ /******************
+ * PARSE RESPONSE *
+ ******************/
+
+ start = strstr(headerresponse,
+ "\r\nContent-Length: ");
+ if (start == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not find \"Content-Length\" in response.\n");
+ }
+
+ /* move "start" to beginning of value in line; find end of line
+ */
+ start = start + HDstrlen("\r\nContent-Length: ");
+ end = strstr(start, "\r\n");
+ if (end == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not find end of content length line");
+ }
+
+ /* place null terminator at end of numbers
+ */
+ *end = '\0';
+
+ content_length = strtoumax((const char *)start,
+ NULL,
+ 0);
+
+ if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX) {
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t\n");
+ }
+
+ if (content_length == 0 ||
+ errno == ERANGE) /* errno set by strtoumax*/
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not convert found \"Content-Length\" response (\"%s\")",
+ start); /* range is null-terminated, remember */
+ }
+
+ handle->filesize = (size_t)content_length;
+
+ /**********************
+ * UNDO HEAD SETTINGS *
+ **********************/
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_NOBODY,
+ NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_NOBODY). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HEADERDATA,
+ NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_HEADERDATA). "
+ "(placeholder flags)");
+ }
+
+done:
+ H5MM_xfree(headerresponse);
+ sds.magic += 1; /* set to bad magic */
+
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* H5FD_s3comms_s3r_getsize */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_open()
+ *
+ * Purpose:
+ *
+ * Logically 'open' a file hosted on S3.
+ *
+ * - create new Request Handle
+ * - copy supplied url
+ * - copy authentication info if supplied
+ * - create CURL handle
+ * - fetch size of file
+ * - connect with server and execute HEAD request
+ * - return request handle ready for reads
+ *
+ * To use 'default' port to connect, `port` should be 0.
+ *
+ * To prevent AWS4 authentication, pass null pointer to `region`, `id`,
+ * and `signing_key`.
+ *
+ * Uses `H5FD_s3comms_parse_url()` to validate and parse url input.
+ *
+ * Return:
+ *
+ * - SUCCESS: Pointer to new request handle.
+ * - FAILURE: NULL
+ * - occurs if:
+ * - authentication strings are inconsistent
+ * - must _all_ be null, or have at least `region` and `id`
+ * - url is NULL (no filename)
+ * - unable to parse url (malformed?)
+ * - error while performing `getsize()`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-01
+ *
+ *----------------------------------------------------------------------------
+ */
+s3r_t *
+H5FD_s3comms_s3r_open(const char *url,
+ const char *region,
+ const char *id,
+ const unsigned char *signing_key)
+{
+ size_t tmplen = 0;
+ CURL *curlh = NULL;
+ s3r_t *handle = NULL;
+ parsed_url_t *purl = NULL;
+ s3r_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_open.\n");
+#endif
+
+
+
+ if (url == NULL || url[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "url cannot be null.\n");
+ }
+
+ if (FAIL == H5FD_s3comms_parse_url(url, &purl)) {
+ /* probably a malformed url, but could be internal error */
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCREATE, NULL,
+ "unable to create parsed url structure");
+ }
+ HDassert( purl != NULL ); /* if above passes, this must be true */
+ HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC );
+
+ handle = (s3r_t *)H5MM_malloc(sizeof(s3r_t));
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "could not malloc space for handle.\n");
+ }
+
+ handle->magic = S3COMMS_S3R_MAGIC;
+ handle->purl = purl;
+ handle->filesize = 0;
+ handle->region = NULL;
+ handle->secret_id = NULL;
+ handle->signing_key = NULL;
+ handle->httpverb = NULL;
+
+ /*************************************
+ * RECORD AUTHENTICATION INFORMATION *
+ *************************************/
+
+ if ((region != NULL && *region != '\0') ||
+ (id != NULL && *id != '\0') ||
+ (signing_key != NULL && *signing_key != '\0'))
+ {
+ /* if one exists, all three must exist
+ */
+ if (region == NULL || region[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "region cannot be null.\n");
+ }
+ if (id == NULL || id[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "secret id cannot be null.\n");
+ }
+ if (signing_key == NULL || signing_key[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "signing key cannot be null.\n");
+ }
+
+ /* copy strings
+ */
+ tmplen = HDstrlen(region) + 1;
+ handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen);
+ if (handle->region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "could not malloc space for handle region copy.\n");
+ }
+ HDmemcpy(handle->region, region, tmplen);
+
+ tmplen = HDstrlen(id) + 1;
+ handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen);
+ if (handle->secret_id == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "could not malloc space for handle ID copy.\n");
+ }
+ HDmemcpy(handle->secret_id, id, tmplen);
+
+ tmplen = SHA256_DIGEST_LENGTH;
+ handle->signing_key =
+ (unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen);
+ if (handle->signing_key == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "could not malloc space for handle key copy.\n");
+ }
+ HDmemcpy(handle->signing_key, signing_key, tmplen);
+ } /* if authentication information provided */
+
+ /************************
+ * INITIATE CURL HANDLE *
+ ************************/
+
+ curlh = curl_easy_init();
+
+ if (curlh == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "problem creating curl easy handle!\n");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HTTPGET,
+ 1L) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_HTTPGET). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HTTP_VERSION,
+ CURL_HTTP_VERSION_1_1) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_HTTP_VERSION). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_FAILONERROR,
+ 1L) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_FAILONERROR). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_WRITEFUNCTION,
+ curlwritecallback) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_WRITEFUNCTION). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_URL,
+ url) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_URL). "
+ "(placeholder flags)");
+ }
+
+#if S3COMMS_CURL_VERBOSITY > 1
+ /* CURL will print (to stdout) information for each operation
+ */
+ curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L);
+#endif
+
+ handle->curlhandle = curlh;
+
+ /*******************
+ * OPEN CONNECTION *
+ * * * * * * * * * *
+ * GET FILE SIZE *
+ *******************/
+
+ if (FAIL ==
+ H5FD_s3comms_s3r_getsize(handle) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "problem in H5FD_s3comms_s3r_getsize.\n");
+ }
+
+ /*********************
+ * FINAL PREPARATION *
+ *********************/
+
+ HDassert( handle->httpverb != NULL );
+ HDmemcpy(handle->httpverb, "GET", 4);
+
+ ret_value = handle;
+
+done:
+ if (ret_value == NULL) {
+ if (curlh != NULL) {
+ curl_easy_cleanup(curlh);
+ }
+ if (FAIL == H5FD_s3comms_free_purl(purl)) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "unable to free parsed url structure")
+ }
+ if (handle != NULL) {
+ H5MM_xfree(handle->region);
+ H5MM_xfree(handle->secret_id);
+ H5MM_xfree(handle->signing_key);
+ if (handle->httpverb != NULL) {
+ H5MM_xfree(handle->httpverb);
+ }
+ H5MM_xfree(handle);
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_s3comms_s3r_open */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_read()
+ *
+ * Purpose:
+ *
+ * Read file pointed to by request handle, writing specified
+ * `offset` .. `offset + len` bytes to buffer `dest`.
+ *
+ * If `len` is 0, reads entirety of file starting at `offset`.
+ * If `offset` and `len` are both 0, reads entire file.
+ *
+ * If `offset` or `offset+len` is greater than the file size, read is
+ * aborted and returns `FAIL`.
+ *
+ * Uses configured "curl easy handle" to perform request.
+ *
+ * In event of error, buffer should remain unaltered.
+ *
+ * If handle is set to authorize a request, creates a new (temporary)
+ * HTTP Request object (hrb_t) for generating requisite headers,
+ * which is then translated to a `curl slist` and set in the curl handle
+ * for the request.
+ *
+ * `dest` _may_ be NULL, but no body data will be recorded.
+ *
+ * - In general practice, NULL should never be passed in as `dest`.
+ * - NULL `dest` passed in by internal function `s3r_getsize()`, in
+ * conjunction with CURLOPT_NOBODY to preempt transmission of file data
+ * from server.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-22
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_s3r_read(s3r_t *handle,
+ haddr_t offset,
+ size_t len,
+ void *dest)
+{
+ CURL *curlh = NULL;
+ CURLcode p_status = CURLE_OK;
+ struct curl_slist *curlheaders = NULL;
+ hrb_node_t *headers = NULL;
+ hrb_node_t *node = NULL;
+ struct tm *now = NULL;
+ char *rangebytesstr = NULL;
+ hrb_t *request = NULL;
+ int ret = 0; /* working variable to check */
+ /* return value of HDsnprintf */
+ struct s3r_datastruct *sds = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_read.\n");
+#endif
+
+ /**************************************
+ * ABSOLUTELY NECESSARY SANITY-CHECKS *
+ **************************************/
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n");
+ }
+ if (handle->magic != S3COMMS_S3R_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n");
+ }
+ if (handle->curlhandle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has bad (null) curlhandle.\n")
+ }
+ if (handle->purl == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has bad (null) url.\n")
+ }
+ HDassert( handle->purl->magic == S3COMMS_PARSED_URL_MAGIC );
+ if (offset > handle->filesize || (len + offset) > handle->filesize) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to read past EoF")
+ }
+
+ curlh = handle->curlhandle;
+
+ /*********************
+ * PREPARE WRITEDATA *
+ *********************/
+
+ if (dest != NULL) {
+ sds = (struct s3r_datastruct *)H5MM_malloc(
+ sizeof(struct s3r_datastruct));
+ if (sds == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "could not malloc destination datastructure.\n");
+ }
+
+ sds->magic = S3COMMS_CALLBACK_DATASTRUCT_MAGIC;
+ sds->data = (char *)dest;
+ sds->size = 0;
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_WRITEDATA,
+ sds) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL,
+ "error while setting CURL option (CURLOPT_WRITEDATA). "
+ "(placeholder flags)");
+ }
+ }
+
+ /*********************
+ * FORMAT HTTP RANGE *
+ *********************/
+
+ if (len > 0) {
+ rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \
+ (S3COMMS_MAX_RANGE_STRING_SIZE+1) );
+ if (rangebytesstr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "could not malloc range format string.\n");
+ }
+ ret = HDsnprintf(rangebytesstr,
+ (S3COMMS_MAX_RANGE_STRING_SIZE),
+ "bytes="H5_PRINTF_HADDR_FMT"-"H5_PRINTF_HADDR_FMT,
+ offset,
+ offset + len - 1);
+ if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format HTTP Range value");
+ } else if (offset > 0) {
+ rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \
+ (S3COMMS_MAX_RANGE_STRING_SIZE+1));
+ if (rangebytesstr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "could not malloc range format string.\n");
+ }
+ ret = HDsnprintf(rangebytesstr,
+ (S3COMMS_MAX_RANGE_STRING_SIZE),
+ "bytes="H5_PRINTF_HADDR_FMT"-",
+ offset);
+ if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format HTTP Range value");
+ }
+
+ /*******************
+ * COMPILE REQUEST *
+ *******************/
+
+ if (handle->signing_key == NULL) {
+ /* Do not authenticate.
+ */
+ if (rangebytesstr != NULL) {
+ /* Pass in range directly
+ */
+ char *bytesrange_ptr = NULL; /* pointer past "bytes=" portion */
+
+ bytesrange_ptr = strchr(rangebytesstr, '=');
+ HDassert( bytesrange_ptr != NULL );
+ bytesrange_ptr++; /* move to first char past '=' */
+ HDassert( *bytesrange_ptr != '\0' );
+
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_RANGE,
+ bytesrange_ptr) )
+ {
+ HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL,
+ "error while setting CURL option (CURLOPT_RANGE). ");
+ }
+ }
+ } else {
+ /* authenticate request
+ */
+ char authorization[512+1];
+ /* 512 := approximate max length...
+ * 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request,"
+ * "SignedHeaders=,Signature=")>
+ * + 8 <yyyyMMDD>
+ * + 64 <hex(sha256())>
+ * + 128 <max? len(secret_id)>
+ * + 20 <max? len(region)>
+ * + 128 <max? len(signed_headers)>
+ */
+ char buffer1[512+1]; /* -> Canonical Request -> Signature */
+ char buffer2[256+1]; /* -> String To Sign -> Credential */
+ char iso8601now[ISO8601_SIZE];
+ char signed_headers[48+1];
+ /* should be large enough for nominal listing:
+ * "host;range;x-amz-content-sha256;x-amz-date"
+ * + '\0', with "range;" possibly absent
+ */
+
+ /* zero start of strings */
+ authorization[0] = 0;
+ buffer1[0] = 0;
+ buffer2[0] = 0;
+ iso8601now[0] = 0;
+ signed_headers[0] = 0;
+
+ /**** VERIFY INFORMATION EXISTS ****/
+
+ if (handle->region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null region.\n");
+ }
+ if (handle->secret_id == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null secret_id.\n");
+ }
+ if (handle->signing_key == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null signing_key.\n");
+ }
+ if (handle->httpverb == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null httpverb.\n");
+ }
+ if (handle->purl->host == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null host.\n");
+ }
+ if (handle->purl->path == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null resource.\n");
+ }
+
+ /**** CREATE HTTP REQUEST STRUCTURE (hrb_t) ****/
+
+ request = H5FD_s3comms_hrb_init_request(
+ (const char *)handle->httpverb,
+ (const char *)handle->purl->path,
+ "HTTP/1.1");
+ if (request == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not allocate hrb_t request.\n");
+ }
+ HDassert( request->magic == S3COMMS_HRB_MAGIC );
+
+ now = gmnow();
+ if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not format ISO8601 time.\n");
+ }
+
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "x-amz-date",
+ (const char *)iso8601now) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set x-amz-date header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "x-amz-content-sha256",
+ (const char *)EMPTY_SHA256) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set x-amz-content-sha256 header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+
+ if (rangebytesstr != NULL) {
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "Range",
+ (const char *)rangebytesstr) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set range header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+ }
+
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "Host",
+ (const char *)handle->purl->host) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set host header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+
+ request->first_header = headers;
+
+ /**** COMPUTE AUTHORIZATION ****/
+
+ if (FAIL == /* buffer1 -> canonical request */
+ H5FD_s3comms_aws_canonical_request(
+ buffer1,
+ 512,
+ signed_headers,
+ 48,
+ request) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "(placeholder flags)\n");
+ }
+ if ( FAIL == /* buffer2->string-to-sign */
+ H5FD_s3comms_tostringtosign(buffer2,
+ buffer1,
+ iso8601now,
+ handle->region) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "(placeholder flags)\n");
+ }
+ if (FAIL == /* buffer1 -> signature */
+ H5FD_s3comms_HMAC_SHA256(handle->signing_key,
+ SHA256_DIGEST_LENGTH,
+ buffer2,
+ HDstrlen(buffer2),
+ buffer1) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "(placeholder flags)\n");
+ }
+
+ iso8601now[8] = 0; /* trim to yyyyMMDD */
+ ret = S3COMMS_FORMAT_CREDENTIAL(buffer2,
+ handle->secret_id,
+ iso8601now,
+ handle->region,
+ "s3");
+ if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format aws4 credential string");
+
+ ret = HDsnprintf(
+ authorization,
+ 512,
+ "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s",
+ buffer2,
+ signed_headers,
+ buffer1);
+ if (ret <= 0 || ret >= 512) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format aws4 authorization string");
+ }
+
+ /* append authorization header to http request buffer
+ */
+ if (H5FD_s3comms_hrb_node_set(
+ &headers,
+ "Authorization",
+ (const char *)authorization)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set Authorization header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+
+ /* update hrb's "first header" pointer
+ */
+ request->first_header = headers;
+
+ /**** SET CURLHANDLE HTTP HEADERS FROM GENERATED DATA ****/
+
+ node = request->first_header;
+ while (node != NULL) {
+ HDassert( node->magic == S3COMMS_HRB_NODE_MAGIC );
+ curlheaders = curl_slist_append(curlheaders,
+ (const char *)node->cat);
+ if (curlheaders == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not append header to curl slist. "
+ "(placeholder flags)\n");
+ }
+ node = node->next;
+ }
+
+ /* sanity-check
+ */
+ if (curlheaders == NULL) {
+ /* above loop was probably never run */
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "curlheaders was never populated.\n");
+ }
+
+ /* finally, set http headers in curl handle
+ */
+ if (curl_easy_setopt(
+ curlh,
+ CURLOPT_HTTPHEADER,
+ curlheaders)
+ != CURLE_OK)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option "
+ "(CURLOPT_HTTPHEADER). (placeholder flags)");
+ }
+
+ } /* end if should authenticate (info provided) */
+
+ /*******************
+ * PERFORM REQUEST *
+ *******************/
+
+#if S3COMMS_CURL_VERBOSITY > 0
+ /* In event of error, print detailed information to stderr
+ * This is not the default behavior.
+ */
+ {
+ long int httpcode = 0;
+ char curlerrbuf[CURL_ERROR_SIZE];
+ curlerrbuf[0] = '\0';
+
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, curlerrbuf) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem setting error buffer")
+ }
+
+ p_status = curl_easy_perform(curlh);
+
+ if (p_status != CURLE_OK) {
+ if (CURLE_OK !=
+ curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem getting response code")
+ }
+ HDfprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %d\n",
+ p_status, httpcode);
+ HDfprintf(stderr, "%s\n", curl_easy_strerror(p_status));
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL,
+ "problem while performing request.\n");
+ }
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem unsetting error buffer")
+ }
+ } /* verbose error reporting */
+#else
+ p_status = curl_easy_perform(curlh);
+
+ if (p_status != CURLE_OK) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL,
+ "curl cannot perform request\n")
+ }
+#endif
+
+#if S3COMMS_DEBUG
+ if (dest != NULL) {
+ HDfprintf(stderr, "len: %d\n", (int)len);
+ HDfprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n");
+ if (sds == NULL) {
+ HDfprintf(stderr, "sds is NULL!\n");
+ }
+ else {
+ HDfprintf(stderr, "sds: 0x%lx\n", (long long)sds);
+ HDfprintf(stderr, "sds->size: %d\n", (int)sds->size);
+ if (len > sds->size) {
+ HDfprintf(stderr, "buffer overwrite\n");
+ }
+ }
+ }
+ else {
+ HDfprintf(stderr, "performed on entire file\n");
+ }
+#endif
+
+done:
+ /* clean any malloc'd resources
+ */
+ if (curlheaders != NULL) {
+ curl_slist_free_all(curlheaders);
+ curlheaders = NULL;
+ }
+ if (rangebytesstr != NULL) {
+ H5MM_xfree(rangebytesstr);
+ rangebytesstr = NULL;
+ }
+ if (sds != NULL) {
+ H5MM_xfree(sds);
+ sds = NULL;
+ }
+ if (request != NULL) {
+ while (headers != NULL)
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(&headers, headers->name, NULL))
+ {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot release header node")
+ }
+ HDassert( NULL == headers );
+ if (FAIL == H5FD_s3comms_hrb_destroy(&request)) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot release header request structure")
+ }
+ HDassert( NULL == request );
+ }
+
+ if (curlh != NULL) {
+ /* clear any Range */
+ if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot unset CURLOPT_RANGE")
+ }
+
+ /* clear headers */
+ if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot unset CURLOPT_HTTPHEADER")
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* H5FD_s3comms_s3r_read */
+
+
+/****************************************************************************
+ * MISCELLANEOUS FUNCTIONS
+ ****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: gmnow()
+ *
+ * Purpose:
+ *
+ * Get the output of `time.h`'s `gmtime()` call while minimizing setup
+ * clutter where important.
+ *
+ * Return:
+ *
+ * Pointer to resulting `struct tm`,as created by gmtime(time_t * T).
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-12
+ *
+ *----------------------------------------------------------------------------
+ */
+struct tm *
+gmnow(void)
+{
+ time_t now;
+ time_t *now_ptr = &now;
+ struct tm *ret_value = NULL;
+
+ /* Doctor assert, checks against error in time() */
+ if ( (time_t)(-1) != time(now_ptr) ) {
+ ret_value = gmtime(now_ptr);
+ }
+
+ HDassert( ret_value != NULL );
+
+ return ret_value;
+} /* end gmnow() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_aws_canonical_request()
+ *
+ * Purpose:
+ *
+ * Compose AWS "Canonical Request" (and signed headers string)
+ * as defined in the REST API documentation.
+ *
+ * Both destination strings are null-terminated.
+ *
+ * Destination string arguments must be provided with adequate space.
+ *
+ * Canonical Request format:
+ *
+ * <HTTP VERB>"\n"
+ * <resource path>"\n"
+ * <query string>"\n"
+ * <header1>"\n" (`lowercase(name)`":"`trim(value)`)
+ * <header2>"\n"
+ * ... (headers sorted by name)
+ * <header_n>"\n"
+ * "\n"
+ * <signed headers>"\n" (`lowercase(header 1 name)`";"`header 2 name`;...)
+ * <hex-string of sha256sum of body> ("e3b0c4429...", e.g.)
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - writes canonical request to respective `...dest` strings
+ * - FAILURE: `FAIL`
+ * - one or more input argument was NULL
+ * - internal error
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-04
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_aws_canonical_request(
+ char *canonical_request_dest,
+ int _cr_size,
+ char *signed_headers_dest,
+ int _sh_size,
+ hrb_t *http_request)
+{
+ hrb_node_t *node = NULL;
+ const char *query_params = ""; /* unused at present */
+ herr_t ret_value = SUCCEED;
+ int ret = 0;
+ size_t cr_size = (size_t)_cr_size;
+ size_t sh_size = (size_t)_sh_size;
+ size_t cr_len = 0; /* working length of canonical request str */
+ size_t sh_len = 0; /* working length of signed headers str */
+ char tmpstr[256+1];
+ tmpstr[256] = 0; /* terminating NULL */
+
+ /* "query params" refers to the optional element in the URL, e.g.
+ * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J
+ * ^-----------------^
+ *
+ * Not handled/implemented as of 2017-10-xx.
+ * Element introduced as empty placeholder and reminder.
+ * Further research to be done if this is ever relevant for the
+ * VFD use-cases.
+ */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_aws_canonical_request.\n");
+#endif
+
+ if (http_request == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "hrb object cannot be null.\n");
+ }
+ HDassert( http_request->magic == S3COMMS_HRB_MAGIC );
+
+ if (canonical_request_dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "canonical request destination cannot be null.\n");
+ }
+
+ if (signed_headers_dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "signed headers destination cannot be null.\n");
+ }
+
+ /* HTTP verb, resource path, and query string lines
+ */
+ cr_len = (HDstrlen(http_request->verb) +
+ HDstrlen(http_request->resource) +
+ HDstrlen(query_params) +
+ (size_t)3); /* three newline chars */
+ if (cr_len >= cr_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "not enough space in canonical request");
+ }
+ ret = HDsnprintf( /* TODO: compiler warning */
+ canonical_request_dest,
+ (cr_size-1),
+ "%s\n%s\n%s\n",
+ http_request->verb,
+ http_request->resource,
+ query_params);
+ if (ret < 0 || (size_t)ret >= cr_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to compose canonical request first line");
+ }
+
+ /* write in canonical headers, building signed headers concurrently
+ */
+ node = http_request->first_header; /* assumed sorted */
+ while (node != NULL) {
+
+ HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC);
+
+ ret = HDsnprintf(
+ tmpstr,
+ 256,
+ "%s:%s\n",
+ node->lowername,
+ node->value);
+ if (ret < 0 || ret >= 256) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to concatenate HTTP header %s:%s",
+ node->lowername,
+ node->value);
+ }
+ cr_len += HDstrlen(tmpstr);
+ if (cr_len + 1 > cr_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "not enough space in canonical request");
+ }
+ strcat(canonical_request_dest, tmpstr);
+
+ ret = HDsnprintf(
+ tmpstr,
+ 256,
+ "%s;",
+ node->lowername);
+ if (ret < 0 || ret >= 256) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to append semicolon to lowername %s",
+ node->lowername);
+ }
+ sh_len += HDstrlen(tmpstr);
+ if (sh_len + 1 > sh_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "not enough space in signed headers");
+ }
+ strcat(signed_headers_dest, tmpstr);
+
+ node = node->next;
+ } /* end while node is not NULL */
+
+ /* remove tailing ';' from signed headers sequence
+ */
+ signed_headers_dest[HDstrlen(signed_headers_dest) - 1] = '\0';
+
+ /* append signed headers and payload hash
+ * NOTE: at present, no HTTP body is handled, per the nature of
+ * requests/range-gets
+ */
+ strcat(canonical_request_dest, "\n");
+ strcat(canonical_request_dest, signed_headers_dest);
+ strcat(canonical_request_dest, "\n");
+ strcat(canonical_request_dest, EMPTY_SHA256);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_aws_canonical_request() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_bytes_to_hex()
+ *
+ * Purpose:
+ *
+ * Produce human-readable hex string [0-9A-F] from sequence of bytes.
+ *
+ * For each byte (char), writes two-character hexadecimal representation.
+ *
+ * No null-terminator appended.
+ *
+ * Assumes `dest` is allocated to enough size (msg_len * 2).
+ *
+ * Fails if either `dest` or `msg` are null.
+ *
+ * `msg_len` message length of 0 has no effect.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - hex string written to `dest` (not null-terminated)
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ * - `msg == NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-12
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_bytes_to_hex(
+ char *dest,
+ const unsigned char *msg,
+ size_t msg_len,
+ hbool_t lowercase)
+{
+ size_t i = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_bytes_to_hex.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "hex destination cannot be null.\n")
+ }
+ if (msg == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "bytes sequence cannot be null.\n")
+ }
+
+ for (i = 0; i < msg_len; i++) {
+ int chars_written = HDsnprintf(&(dest[i * 2]),
+ 3, /* 'X', 'X', '\n' */
+ (lowercase == TRUE) ? "%02x" : "%02X",
+ msg[i]);
+ if (chars_written != 2) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem while writing hex chars for %c",
+ msg[i]);
+ }
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* end H5FD_s3comms_bytes_to_hex() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_free_purl()
+ *
+ * Purpose:
+ *
+ * Release resources from a parsed_url_t pointer.
+ *
+ * If pointer is null, nothing happens.
+ *
+ * Return:
+ *
+ * `SUCCEED` (never fails)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-01
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_free_purl(parsed_url_t *purl)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if S3COMMS_DEBUG
+ HDprintf("called H5FD_s3comms_free_purl.\n");
+#endif
+
+ if (purl != NULL) {
+ HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC );
+ if (purl->scheme != NULL) { H5MM_xfree(purl->scheme); }
+ if (purl->host != NULL) { H5MM_xfree(purl->host); }
+ if (purl->port != NULL) { H5MM_xfree(purl->port); }
+ if (purl->path != NULL) { H5MM_xfree(purl->path); }
+ if (purl->query != NULL) { H5MM_xfree(purl->query); }
+ purl->magic += 1ul;
+ H5MM_xfree(purl);
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD_s3comms_free_purl() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_HMAC_SHA256()
+ *
+ * Purpose:
+ *
+ * Generate Hash-based Message Authentication Checksum using the SHA-256
+ * hashing algorithm.
+ *
+ * Given a key, message, and respective lengths (to accommodate null
+ * characters in either), generate _hex string_ of authentication checksum
+ * and write to `dest`.
+ *
+ * `dest` must be at least `SHA256_DIGEST_LENGTH * 2` characters in size.
+ * Not enforceable by this function.
+ * `dest` will _not_ be null-terminated by this function.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - hex string written to `dest` (not null-terminated)
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ * - error while generating hex string output
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-??
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_HMAC_SHA256(
+ const unsigned char *key,
+ size_t key_len,
+ const char *msg,
+ size_t msg_len,
+ char *dest)
+{
+ unsigned char md[SHA256_DIGEST_LENGTH];
+ unsigned int md_len = SHA256_DIGEST_LENGTH;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_HMAC_SHA256.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be null.");
+ }
+
+ HMAC(EVP_sha256(),
+ key,
+ (int)key_len,
+ (const unsigned char *)msg,
+ msg_len,
+ md,
+ &md_len);
+
+ if (H5FD_s3comms_bytes_to_hex(
+ dest,
+ (const unsigned char *)md,
+ (size_t)md_len,
+ true)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not convert to hex string.");
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* H5FD_s3comms_HMAC_SHA256 */
+
+
+/*-----------------------------------------------------------------------------
+ *
+ * Function: H5FD__s3comms_load_aws_creds_from_file()
+ *
+ * Purpose:
+ *
+ * Extract AWS configuration information from a target file.
+ *
+ * Given a file and a profile name, e.g. "ros3_vfd_test", attempt to locate
+ * that region in the file. If not found, returns in error and output
+ * pointers are not modified.
+ *
+ * If the profile label is found, attempts to locate and parse configuration
+ * data, stopping at the first line where:
+ * + reached end of file
+ * + line does not start with a recognized setting name
+ *
+ * Following AWS documentation, looks for any of:
+ * + aws_access_key_id
+ * + aws_secret_access_key
+ * + region
+ *
+ * To be valid, the setting must begin the line with one of the keywords,
+ * followed immediately by an equals sign '=', and have some data before
+ * newline at end of line.
+ * + `spam=eggs` would be INVALID because name is unrecognized
+ * + `region = us-east-2` would be INVALID because of spaces
+ * + `region=` would be INVALID because no data.
+ *
+ * Upon successful parsing of a setting line, will store the result in the
+ * corresponding output pointer. If the output pointer is NULL, will skip
+ * any matching setting line while parsing -- useful to prevent overwrite
+ * when reading from multiple files.
+ *
+ * Return:
+ *
+ * + SUCCESS: `SUCCEED`
+ * + no error. settings may or may not have been loaded.
+ * + FAILURE: `FAIL`
+ * + internal error occurred.
+ * + -1 :: unable to format profile label
+ * + -2 :: profile name/label not found in file
+ * + -3 :: some other error
+ *
+ * Programmer: Jacob Smith
+ * 2018-02-27
+ *
+ *-----------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__s3comms_load_aws_creds_from_file(
+ FILE *file,
+ const char *profile_name,
+ char *key_id,
+ char *access_key,
+ char *aws_region)
+{
+ char profile_line[32];
+ char buffer[128];
+ const char *setting_names[] = {
+ "region",
+ "aws_access_key_id",
+ "aws_secret_access_key",
+ };
+ char * const setting_pointers[] = {
+ aws_region,
+ key_id,
+ access_key,
+ };
+ unsigned setting_count = 3;
+ herr_t ret_value = SUCCEED;
+ unsigned buffer_i = 0;
+ unsigned setting_i = 0;
+ int found_setting = 0;
+ char *line_buffer = &(buffer[0]);
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called load_aws_creds_from_file.\n");
+#endif
+
+ /* format target line for start of profile */
+ if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format profile label")
+ }
+
+ /* look for start of profile */
+ do {
+ /* clear buffer */
+ for (buffer_i=0; buffer_i < 128; buffer_i++) {
+ buffer[buffer_i] = 0;
+ }
+
+ line_buffer = fgets(line_buffer, 128, file);
+ if (line_buffer == NULL) { /* reached end of file */
+ goto done;
+ }
+ } while (strncmp(line_buffer, profile_line, HDstrlen(profile_line)));
+
+ /* extract credentials from lines */
+ do {
+ /* clear buffer */
+ for (buffer_i=0; buffer_i < 128; buffer_i++) {
+ buffer[buffer_i] = 0;
+ }
+
+ /* collect a line from file */
+ line_buffer = fgets(line_buffer, 128, file);
+ if (line_buffer == NULL) {
+ goto done; /* end of file */
+ }
+
+ /* loop over names to see if line looks like assignment */
+ for (setting_i = 0; setting_i < setting_count; setting_i++) {
+ size_t setting_name_len = 0;
+ const char *setting_name = NULL;
+ char line_prefix[128];
+
+ setting_name = setting_names[setting_i];
+ setting_name_len = HDstrlen(setting_name);
+ if (HDsnprintf(line_prefix, 128, "%s=", setting_name) < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format line prefix")
+ }
+
+ /* found a matching name? */
+ if (!HDstrncmp(line_buffer, line_prefix, setting_name_len + 1)) {
+ found_setting = 1;
+
+ /* skip NULL destination buffer */
+ if (setting_pointers[setting_i] == NULL) {
+ break;
+ }
+
+ /* advance to end of name in string */
+ do {
+ line_buffer++;
+ } while (*line_buffer != 0 && *line_buffer != '=');
+
+ if (*line_buffer == 0 || *(line_buffer+1) == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "incomplete assignment in file")
+ }
+ line_buffer++; /* was pointing at '='; advance */
+
+ /* copy line buffer into out pointer */
+ if (HDstrncpy(
+ setting_pointers[setting_i],
+ (const char *)line_buffer,
+ HDstrlen(line_buffer))
+ == NULL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to copy line into pointer")
+ }
+
+ /* "trim" tailing whitespace by replacing with null terminator*/
+ buffer_i = 0;
+ while (!isspace(setting_pointers[setting_i][buffer_i])) {
+ buffer_i++;
+ }
+ setting_pointers[setting_i][buffer_i] = '\0';
+
+ break; /* have read setting; don't compare with others */
+ } /* end if possible name match */
+ } /* end for each setting name */
+ } while (found_setting);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD__s3comms_load_aws_creds_from_file() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_load_aws_profile()
+ *
+ * Purpose :
+ *
+ * Read aws profile elements from standard location on system and store
+ * settings in memory.
+ *
+ * Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard
+ * files for AWS tools. If a file exists (can be opened), looks for the
+ * given profile name and reads the settings into the relevant buffer.
+ *
+ * Any setting duplicated in both files will be set to that from
+ * `credentials`.
+ *
+ * Settings are stored in the supplied buffers as null-terminated strings.
+ *
+ * Return:
+ *
+ * + SUCCESS: `SUCCEED` (0)
+ * + no error occurred and all settings were populated
+ * + FAILURE: `FAIL` (-1)
+ * + internal error occurred
+ * + unable to locate profile
+ * + region, key id, and secret key were not all found and set
+ *
+ * Programmer: Jacob Smith
+ * 2018-02-27
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_load_aws_profile(const char *profile_name,
+ char *key_id_out,
+ char *secret_access_key_out,
+ char *aws_region_out)
+{
+ herr_t ret_value = SUCCEED;
+ FILE *credfile = NULL;
+ char awspath[117];
+ char filepath[128];
+ int ret = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_load_aws_profile.\n");
+#endif
+
+#ifdef H5_HAVE_WIN32_API
+ ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")) ;
+#else
+ ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")) ;
+#endif
+ if (ret < 0 || (size_t)ret >= 117) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format home-aws path")
+ }
+ ret = HDsnprintf(filepath, 128, "%s%s", awspath, "credentials");
+ if (ret < 0 || (size_t)ret >= 128) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format credentials path")
+ }
+
+ credfile = fopen(filepath, "r");
+ if (credfile != NULL) {
+ if (H5FD__s3comms_load_aws_creds_from_file(
+ credfile,
+ profile_name,
+ key_id_out,
+ secret_access_key_out,
+ aws_region_out)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to load from aws credentials")
+ }
+ if (fclose(credfile) == EOF) {
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close credentials file")
+ }
+ credfile = NULL;
+ } /* end if credential file opened */
+
+ ret = HDsnprintf(filepath, 128, "%s%s", awspath, "config");
+ if (ret < 0 || (size_t)ret >= 128) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format config path")
+ }
+ credfile = fopen(filepath, "r");
+ if (credfile != NULL) {
+ if (H5FD__s3comms_load_aws_creds_from_file(
+ credfile,
+ profile_name,
+ (*key_id_out == 0) ? key_id_out : NULL,
+ (*secret_access_key_out == 0) ? secret_access_key_out : NULL,
+ (*aws_region_out == 0) ? aws_region_out : NULL)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to load from aws config")
+ }
+ if (fclose(credfile) == EOF) {
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close config file")
+ }
+ credfile = NULL;
+ } /* end if credential file opened */
+
+ /* fail if not all three settings were loaded */
+ if (*key_id_out == 0 ||
+ *secret_access_key_out == 0 ||
+ *aws_region_out == 0)
+ {
+ ret_value = FAIL;
+ }
+
+done:
+ if (credfile != NULL) {
+ if (fclose(credfile) == EOF) {
+ HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL,
+ "problem error-closing aws configuration file")
+ }
+ }
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_load_aws_profile() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_nlowercase()
+ *
+ * Purpose:
+ *
+ * From string starting at `s`, write `len` characters to `dest`,
+ * converting all to lowercase.
+ *
+ * Behavior is undefined if `s` is NULL or `len` overruns the allocated
+ * space of either `s` or `dest`.
+ *
+ * Provided as convenience.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - upon completion, `dest` is populated
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-18
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_nlowercase(
+ char *dest,
+ const char *s,
+ size_t len)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_nlowercase.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be null.\n");
+ }
+
+ if (len > 0) {
+ HDmemcpy(dest, s, len);
+ do {
+ len--;
+ dest[len] = (char)tolower( (int)dest[len] );
+ } while (len > 0);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_nlowercase() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_parse_url()
+ *
+ * Purpose:
+ *
+ * Parse URL-like string and stuff URL components into
+ * `parsed_url` structure, if possible.
+ *
+ * Expects null-terminated string of format:
+ * SCHEME "://" HOST [":" PORT ] ["/" [ PATH ] ] ["?" QUERY]
+ * where SCHEME :: "[a-zA-Z/.-]+"
+ * PORT :: "[0-9]"
+ *
+ * Stores resulting structure in argument pointer `purl`, if successful,
+ * creating and populating new `parsed_url_t` structure pointer.
+ * Empty or absent elements are NULL in new purl structure.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - `purl` pointer is populated
+ * - FAILURE: `FAIL`
+ * - unable to parse
+ * - `purl` is unaltered (probably NULL)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-30
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_parse_url(
+ const char *str,
+ parsed_url_t **_purl)
+{
+ parsed_url_t *purl = NULL; /* pointer to new structure */
+ const char *tmpstr = NULL; /* working pointer in string */
+ const char *curstr = str; /* "start" pointer in string */
+ long int len = 0; /* substring length */
+ long int urllen = 0; /* length of passed-in url string */
+ unsigned int i = 0;
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_NOAPI_NOINIT;
+
+#if S3COMMS_DEBUG
+ HDprintf("called H5FD_s3comms_parse_url.\n");
+#endif
+
+ if (str == NULL || *str == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid url string");
+ }
+
+ urllen = (long int)HDstrlen(str);
+
+ purl = (parsed_url_t *)H5MM_malloc(sizeof(parsed_url_t));
+ if (purl == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for parsed_url_t");
+ }
+ purl->magic = S3COMMS_PARSED_URL_MAGIC;
+ purl->scheme = NULL;
+ purl->host = NULL;
+ purl->port = NULL;
+ purl->path = NULL;
+ purl->query = NULL;
+
+ /***************
+ * READ SCHEME *
+ ***************/
+
+ tmpstr = HDstrchr(curstr, ':');
+ if (tmpstr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid SCHEME construction: probably not URL");
+ }
+ len = tmpstr - curstr;
+ HDassert( (0 <= len) && (len < urllen) );
+
+ /* check for restrictions
+ */
+ for (i = 0; i < len; i++) {
+ /* scheme = [a-zA-Z+-.]+ (terminated by ":") */
+ if (!isalpha(curstr[i]) &&
+ '+' != curstr[i] &&
+ '-' != curstr[i] &&
+ '.' != curstr[i])
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid SCHEME construction");
+ }
+ }
+ /* copy lowercased scheme to structure
+ */
+ purl->scheme = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->scheme == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for SCHEME");
+ }
+ (void)HDstrncpy(purl->scheme, curstr, (size_t)len);
+ purl->scheme[len] = '\0';
+ for ( i = 0; i < len; i++ ) {
+ purl->scheme[i] = (char)tolower(purl->scheme[i]);
+ }
+
+ /* Skip "://" */
+ tmpstr += 3;
+ curstr = tmpstr;
+
+ /*************
+ * READ HOST *
+ *************/
+
+ if (*curstr == '[') {
+ /* IPv6 */
+ while (']' != *tmpstr) {
+ if (tmpstr == 0) { /* end of string reached! */
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "reached end of URL: incomplete IPv6 HOST");
+ }
+ tmpstr++;
+ }
+ tmpstr++;
+ } /* end if (IPv6) */
+ else {
+ while (0 != *tmpstr) {
+ if (':' == *tmpstr ||
+ '/' == *tmpstr ||
+ '?' == *tmpstr)
+ {
+ break;
+ }
+ tmpstr++;
+ }
+ } /* end else (IPv4) */
+ len = tmpstr - curstr;
+ if (len == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "HOST substring cannot be empty");
+ }
+ else
+ if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of HOST substring");
+ }
+
+ /* copy host
+ */
+ purl->host = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->host == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for HOST");
+ }
+ (void)HDstrncpy(purl->host, curstr, (size_t)len);
+ purl->host[len] = 0;
+
+ /*************
+ * READ PORT *
+ *************/
+
+ if (':' == *tmpstr) {
+ tmpstr += 1; /* advance past ':' */
+ curstr = tmpstr;
+ while ((0 != *tmpstr) && ('/' != *tmpstr) && ('?' != *tmpstr)) {
+ tmpstr++;
+ }
+ len = tmpstr - curstr;
+ if (len == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "PORT element cannot be empty");
+ }
+ else
+ if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of PORT substring");
+ }
+ for (i = 0; i < len; i ++) {
+ if (!isdigit(curstr[i])) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "PORT is not a decimal string");
+ }
+ }
+
+ /* copy port
+ */
+ purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->port == NULL) { /* cannot malloc */
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for PORT");
+ }
+ (void)HDstrncpy(purl->port, curstr, (size_t)len);
+ purl->port[len] = 0;
+ } /* end if PORT element */
+
+ /*************
+ * READ PATH *
+ *************/
+
+ if ('/' == *tmpstr) {
+ /* advance past '/' */
+ tmpstr += 1;
+ curstr = tmpstr;
+
+ /* seek end of PATH
+ */
+ while ((0 != *tmpstr) && ('?' != *tmpstr)) {
+ tmpstr++;
+ }
+ len = tmpstr - curstr;
+ if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of PATH substring");
+ }
+ if (len > 0) {
+ purl->path = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->path == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for PATH");
+ } /* cannot malloc path pointer */
+ (void)HDstrncpy(purl->path, curstr, (size_t)len);
+ purl->path[len] = 0;
+ }
+ } /* end if PATH element */
+
+ /**************
+ * READ QUERY *
+ **************/
+
+ if ('?' == *tmpstr) {
+ tmpstr += 1;
+ curstr = tmpstr;
+ while (0 != *tmpstr) {
+ tmpstr++;
+ }
+ len = tmpstr - curstr;
+ if (len == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "QUERY cannot be empty");
+ } else if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of QUERY substring");
+ }
+ purl->query = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->query == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for QUERY");
+ } /* cannot malloc path pointer */
+ (void)HDstrncpy(purl->query, curstr, (size_t)len);
+ purl->query[len] = 0;
+ } /* end if QUERY exists */
+
+ *_purl = purl;
+ ret_value = SUCCEED;
+
+done:
+ if (ret_value == FAIL) {
+ H5FD_s3comms_free_purl(purl);
+ }
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_parse_url() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_percent_encode_char()
+ *
+ * Purpose:
+ *
+ * "Percent-encode" utf-8 character `c`, e.g.,
+ * '$' -> "%24"
+ * '¢' -> "%C2%A2"
+ *
+ * `c` cannot be null.
+ *
+ * Does not (currently) accept multi-byte characters...
+ * limit to (?) u+00ff, well below upper bound for two-byte utf-8 encoding
+ * (u+0080..u+07ff).
+ *
+ * Writes output to `repr`.
+ * `repr` cannot be null.
+ * Assumes adequate space i `repr`...
+ * >>> char[4] or [7] for most characters,
+ * >>> [13] as theoretical maximum.
+ *
+ * Representation `repr` is null-terminated.
+ *
+ * Stores length of representation (without null terminator) at pointer
+ * `repr_len`.
+ *
+ * Return : SUCCEED/FAIL
+ *
+ * - SUCCESS: `SUCCEED`
+ * - percent-encoded representation written to `repr`
+ * - 'repr' is null-terminated
+ * - FAILURE: `FAIL`
+ * - `c` or `repr` was NULL
+ *
+ * Programmer: Jacob Smith
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_percent_encode_char(
+ char *repr,
+ const unsigned char c,
+ size_t *repr_len)
+{
+ unsigned int i = 0;
+ int chars_written = 0;
+ herr_t ret_value = SUCCEED;
+#if S3COMMS_DEBUG
+ unsigned char s[2] = {c, 0};
+ unsigned char hex[3] = {0, 0, 0};
+#endif
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_percent_encode_char.\n");
+#endif
+
+ if (repr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no destination `repr`.\n")
+ }
+
+#if S3COMMS_DEBUG
+ H5FD_s3comms_bytes_to_hex((char *)hex, s, 1, FALSE);
+ HDfprintf(stdout, " CHAR: \'%s\'\n", s);
+ HDfprintf(stdout, " CHAR-HEX: \"%s\"\n", hex);
+#endif
+
+ if (c <= (unsigned char)0x7f) {
+ /* character represented in a single "byte"
+ * and single percent-code
+ */
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, " SINGLE-BYTE\n");
+#endif
+ *repr_len = 3;
+ chars_written = HDsnprintf(repr, 4, "%%%02X", c);
+ if (chars_written < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot write char %c",
+ c);
+ }
+ } /* end if single-byte unicode char */
+ else {
+ /* multi-byte, multi-percent representation
+ */
+ unsigned int acc = 0; /* byte accumulator */
+ unsigned int k = 0; /* uint character representation */
+ unsigned int stack_size = 0;
+ unsigned char stack[4] = {0, 0, 0, 0};
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, " MULTI-BYTE\n");
+#endif
+ stack_size = 0;
+ k = (unsigned int)c;
+ *repr_len = 0;
+ do {
+ /* push number onto stack in six-bit slices
+ */
+ acc = k;
+ acc >>= 6; /* cull least */
+ acc <<= 6; /* six bits */
+ stack[stack_size++] = (unsigned char)(k - acc);
+ k = acc >> 6;
+ } while (k > 0);
+
+ /* `stack` now has two to four six-bit 'numbers' to be put into
+ * UTF-8 byte fields.
+ */
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, " STACK:\n {\n");
+ for (i = 0; i < stack_size; i++) {
+ H5FD_s3comms_bytes_to_hex(
+ (char *)hex,
+ (&stack[i]),
+ 1,
+ FALSE);
+ hex[2] = 0;
+ HDfprintf(stdout, " %s,\n", hex);
+ }
+ HDfprintf(stdout, " }\n");
+#endif
+
+ /****************
+ * leading byte *
+ ****************/
+
+ /* prepend 11[1[1]]0 to first byte */
+ /* 110xxxxx, 1110xxxx, or 11110xxx */
+ acc = 0xC0; /* 0x11000000 */
+ acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */
+ acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */
+ stack_size--;
+ chars_written = HDsnprintf(
+ repr,
+ 4,
+ "%%%02X",
+ (unsigned char)(acc + stack[stack_size]));
+ if (chars_written < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot write char %c",
+ c);
+ }
+ *repr_len += 3;
+
+ /************************
+ * continuation byte(s) *
+ ************************/
+
+ /* 10xxxxxx */
+ for (i = 0; i < stack_size; i++) {
+ chars_written = HDsnprintf(
+ &repr[i*3 + 3],
+ 4,
+ "%%%02X",
+ (unsigned char)(0x80 + stack[stack_size - 1 - i]));
+ if (chars_written < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot write char %c",
+ c);
+ }
+ *repr_len += 3;
+ } /* end for each continuation byte */
+ } /* end else (multi-byte) */
+
+ *(repr + *repr_len) = '\0';
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* H5FD_s3comms_percent_encode_char */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_signing_key()
+ *
+ * Purpose:
+ *
+ * Create AWS4 "Signing Key" from secret key, AWS region, and timestamp.
+ *
+ * Sequentially runs HMAC_SHA256 on strings in specified order,
+ * generating re-usable checksum (according to documentation, valid for
+ * 7 days from time given).
+ *
+ * `secret` is `access key id` for targeted service/bucket/resource.
+ *
+ * `iso8601now` must conform to format, yyyyMMDD'T'hhmmss'Z'
+ * e.g. "19690720T201740Z".
+ *
+ * `region` should be one of AWS service region names, e.g. "us-east-1".
+ *
+ * Hard-coded "service" algorithm requirement to "s3".
+ *
+ * Inputs must be null-terminated strings.
+ *
+ * Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`.
+ * Programmer must ensure that `md` is appropriately allocated.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - raw byte data of signing key written to `md`
+ * - FAILURE: `FAIL`
+ * - if any input arguments was NULL
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-13
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_signing_key(
+ unsigned char *md,
+ const char *secret,
+ const char *region,
+ const char *iso8601now)
+{
+ char *AWS4_secret = NULL;
+ size_t AWS4_secret_len = 0;
+ unsigned char datekey[SHA256_DIGEST_LENGTH];
+ unsigned char dateregionkey[SHA256_DIGEST_LENGTH];
+ unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH];
+ int ret = 0; /* return value of HDsnprintf */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_signing_key.\n");
+#endif
+
+ if (md == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Destination `md` cannot be NULL.\n")
+ }
+ if (secret == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "`secret` cannot be NULL.\n")
+ }
+ if (region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "`region` cannot be NULL.\n")
+ }
+ if (iso8601now == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "`iso8601now` cannot be NULL.\n")
+ }
+
+ AWS4_secret_len = 4 + HDstrlen(secret) + 1;
+ AWS4_secret = (char*)H5MM_malloc(sizeof(char *) * AWS4_secret_len);
+ if (AWS4_secret == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Could not allocate space.\n")
+ }
+
+ /* prepend "AWS4" to start of the secret key
+ */
+ ret = HDsnprintf(AWS4_secret, AWS4_secret_len,"%s%s", "AWS4", secret);
+ if ((size_t)ret != (AWS4_secret_len - 1)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem writing AWS4+secret `%s`",
+ secret);
+ }
+
+ /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest
+ * we know digest length, so ignore via NULL
+ */
+ HMAC(EVP_sha256(),
+ (const unsigned char *)AWS4_secret,
+ (int)HDstrlen(AWS4_secret),
+ (const unsigned char*)iso8601now,
+ 8, /* 8 --> length of 8 --> "yyyyMMDD" */
+ datekey,
+ NULL);
+ HMAC(EVP_sha256(),
+ (const unsigned char *)datekey,
+ SHA256_DIGEST_LENGTH,
+ (const unsigned char *)region,
+ HDstrlen(region),
+ dateregionkey,
+ NULL);
+ HMAC(EVP_sha256(),
+ (const unsigned char *)dateregionkey,
+ SHA256_DIGEST_LENGTH,
+ (const unsigned char *)"s3",
+ 2,
+ dateregionservicekey,
+ NULL);
+ HMAC(EVP_sha256(),
+ (const unsigned char *)dateregionservicekey,
+ SHA256_DIGEST_LENGTH,
+ (const unsigned char *)"aws4_request",
+ 12,
+ md,
+ NULL);
+
+done:
+ H5MM_xfree(AWS4_secret);
+
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_signing_key() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_tostringtosign()
+ *
+ * Purpose:
+ *
+ * Get AWS "String to Sign" from Canonical Request, timestamp,
+ * and AWS "region".
+ *
+ * Common between single request and "chunked upload",
+ * conforms to:
+ * "AWS4-HMAC-SHA256\n" +
+ * <ISO8601 date format> + "\n" + // yyyyMMDD'T'hhmmss'Z'
+ * <yyyyMMDD> + "/" + <AWS Region> + "/s3/aws4-request\n" +
+ * hex(SHA256(<CANONICAL-REQUEST>))
+ *
+ * Inputs `creq` (canonical request string), `now` (ISO8601 format),
+ * and `region` (s3 region designator string) must all be
+ * null-terminated strings.
+ *
+ * Result is written to `dest` with null-terminator.
+ * It is left to programmer to ensure `dest` has adequate space.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - "string to sign" written to `dest` and null-terminated
+ * - FAILURE: `FAIL`
+ * - if any of the inputs are NULL
+ * - if an error is encountered while computing checksum
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-??
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_tostringtosign(
+ char *dest,
+ const char *req,
+ const char *now,
+ const char *region)
+{
+ unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1];
+ size_t d = 0;
+ char day[9];
+ char hexsum[SHA256_DIGEST_LENGTH * 2 + 1];
+ size_t i = 0;
+ int ret = 0; /* HDsnprintf return value */
+ herr_t ret_value = SUCCEED;
+ char tmp[128];
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_tostringtosign.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination buffer cannot be null.\n")
+ }
+ if (req == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "canonical request cannot be null.\n")
+ }
+ if (now == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Timestring cannot be NULL.\n")
+ }
+ if (region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Region cannot be NULL.\n")
+ }
+
+ for (i = 0; i < 128; i++) {
+ tmp[i] = '\0';
+ }
+ for (i = 0; i < SHA256_DIGEST_LENGTH * 2 + 1; i++) {
+ checksum[i] = '\0';
+ hexsum[i] = '\0';
+ }
+ HDstrncpy(day, now, 8);
+ day[8] = '\0';
+ ret = HDsnprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region);
+ if (ret <= 0 || ret >= 127) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem adding day and region to string")
+ }
+
+ HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17);
+ d = 17;
+
+ HDmemcpy((dest+d), now, HDstrlen(now));
+ d += HDstrlen(now);
+ dest[d++] = '\n';
+
+ HDmemcpy((dest + d), tmp, HDstrlen(tmp));
+ d += HDstrlen(tmp);
+ dest[d++] = '\n';
+
+ SHA256((const unsigned char *)req,
+ HDstrlen(req),
+ checksum);
+
+ if (H5FD_s3comms_bytes_to_hex(
+ hexsum,
+ (const unsigned char *)checksum,
+ SHA256_DIGEST_LENGTH,
+ true)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not create hex string");
+ }
+
+ for (i = 0; i < SHA256_DIGEST_LENGTH * 2; i++) {
+ dest[d++] = hexsum[i];
+ }
+
+ dest[d] = '\0';
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5ros3_tostringtosign() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_trim()
+ *
+ * Purpose:
+ *
+ * Remove all whitespace characters from start and end of a string `s`
+ * of length `s_len`, writing trimmed string copy to `dest`.
+ * Stores number of characters remaining at `n_written`.
+ *
+ * Destination for trimmed copy `dest` cannot be null.
+ * `dest` must have adequate space allocated for trimmed copy.
+ * If inadequate space, behavior is undefined, possibly resulting
+ * in segfault or overwrite of other data.
+ *
+ * If `s` is NULL or all whitespace, `dest` is untouched and `n_written`
+ * is set to 0.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-18
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_trim(char *dest,
+ char *s,
+ size_t s_len,
+ size_t *n_written)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_trim.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be null.")
+ }
+ if (s == NULL) {
+ s_len = 0;
+ }
+
+ if (s_len > 0) {
+ /* Find first non-whitespace character from start;
+ * reduce total length per character.
+ */
+ while ((s_len > 0) &&
+ isspace((unsigned char)s[0]) && s_len > 0)
+ {
+ s++;
+ s_len--;
+ }
+
+ /* Find first non-whitespace character from tail;
+ * reduce length per-character.
+ * If length is 0 already, there is no non-whitespace character.
+ */
+ if (s_len > 0) {
+ do {
+ s_len--;
+ } while( isspace((unsigned char)s[s_len]) );
+ s_len++;
+
+ /* write output into dest
+ */
+ HDmemcpy(dest, s, s_len);
+ }
+ }
+
+ *n_written = s_len;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_s3comms_trim() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_uriencode()
+ *
+ * Purpose:
+ *
+ * URIencode (percent-encode) every byte except "[a-zA-Z0-9]-._~".
+ *
+ * For each character in source string `_s` from `s[0]` to `s[s_len-1]`,
+ * writes to `dest` either the raw character or its percent-encoded
+ * equivalent.
+ *
+ * See `H5FD_s3comms_bytes_to_hex` for information on percent-encoding.
+ *
+ * Space (' ') character encoded as "%20" (not "+")
+ *
+ * Forward-slash ('/') encoded as "%2F" only when `encode_slash == true`.
+ *
+ * Records number of characters written at `n_written`.
+ *
+ * Assumes that `dest` has been allocated with enough space.
+ *
+ * Neither `dest` nor `s` can be NULL.
+ *
+ * `s_len == 0` will have no effect.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - source strings `s` or destination `dest` are NULL
+ * - error while attempting to percent-encode a character
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-??
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_uriencode(
+ char *dest,
+ const char *s,
+ size_t s_len,
+ hbool_t encode_slash,
+ size_t *n_written)
+{
+ char c = 0;
+ size_t dest_off = 0;
+ char hex_buffer[13];
+ size_t hex_off = 0;
+ size_t hex_len = 0;
+ herr_t ret_value = SUCCEED;
+ size_t s_off = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "H5FD_s3comms_uriencode called.\n");
+#endif
+
+ if (s == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "source string cannot be NULL");
+ }
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be NULL");
+ }
+
+ /* Write characters to destination, converting to percent-encoded
+ * "hex-utf-8" strings if necessary.
+ * e.g., '$' -> "%24"
+ */
+ for (s_off = 0; s_off < s_len; s_off++) {
+ c = s[s_off];
+ if (isalnum(c) ||
+ c == '.' ||
+ c == '-' ||
+ c == '_' ||
+ c == '~' ||
+ (c == '/' && encode_slash == FALSE))
+ {
+ dest[dest_off++] = c;
+ }
+ else {
+ hex_off = 0;
+ if (H5FD_s3comms_percent_encode_char(
+ hex_buffer,
+ (const unsigned char)c,
+ &hex_len)
+ == FAIL)
+ {
+ hex_buffer[0] = c;
+ hex_buffer[1] = 0;
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to percent-encode character \'%s\' "
+ "at %d in \"%s\"", hex_buffer, (int)s_off, s);
+ }
+
+ for (hex_off = 0; hex_off < hex_len; hex_off++) {
+ dest[dest_off++] = hex_buffer[hex_off];
+ }
+ } /* end else (not a regular character) */
+ } /* end for each character */
+
+ if (dest_off < s_len)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "buffer overflow");
+
+ *n_written = dest_off;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_s3comms_uriencode */
+
+#endif /* H5_HAVE_ROS3_VFD */
+
diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h
new file mode 100644
index 0000000..c9d6c9d
--- /dev/null
+++ b/src/H5FDs3comms.h
@@ -0,0 +1,610 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*****************************************************************************
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * This is the header for the S3 Communications module
+ *
+ * ***NOT A FILE DRIVER***
+ *
+ * Purpose:
+ *
+ * - Provide structures and functions related to communicating with
+ * Amazon S3 (Simple Storage Service).
+ * - Abstract away the REST API (HTTP,
+ * networked communications) behind a series of uniform function calls.
+ * - Handle AWS4 authentication, if appropriate.
+ * - Fail predictably in event of errors.
+ * - Eventually, support more S3 operations, such as creating, writing to,
+ * and removing Objects remotely.
+ *
+ * translates:
+ * `read(some_file, bytes_offset, bytes_length, &dest_buffer);`
+ * to:
+ * ```
+ * GET myfile HTTP/1.1
+ * Host: somewhere.me
+ * Range: bytes=4096-5115
+ * ```
+ * and places received bytes from HTTP response...
+ * ```
+ * HTTP/1.1 206 Partial-Content
+ * Content-Range: 4096-5115/63239
+ *
+ * <bytes>
+ * ```
+ * ...in destination buffer.
+ *
+ * TODO: put documentation in a consistent place and point to it from here.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-30
+ *
+ *****************************************************************************/
+
+#ifndef H5FDs3comms_H
+#define H5FDs3comms_H
+
+
+#include "H5private.h" /* Generic Functions */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/* Necessary S3 headers */
+#include <curl/curl.h>
+#include <openssl/evp.h>
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+/*****************
+ * PUBLIC MACROS *
+ *****************/
+
+/* hexadecimal string of pre-computed sha256 checksum of the empty string
+ * hex(sha256sum(""))
+ */
+#define EMPTY_SHA256 \
+"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+/* string length (plus null terminator)
+ * example ISO8601-format string: "20170713T145903Z" (YYYYmmdd'T'HHMMSS'_')
+ */
+#define ISO8601_SIZE 17
+
+/* string length (plus null terminator)
+ * example RFC7231-format string: "Fri, 30 Jun 2017 20:41:55 GMT"
+ */
+#define RFC7231_SIZE 30
+
+/*---------------------------------------------------------------------------
+ *
+ * Macro: ISO8601NOW()
+ *
+ * Purpose:
+ *
+ * write "YYYYmmdd'T'HHMMSS'Z'" (less single-quotes) to dest
+ * e.g., "20170630T204155Z"
+ *
+ * wrapper for strftime()
+ *
+ * It is left to the programmer to check return value of
+ * ISO8601NOW (should equal ISO8601_SIZE - 1).
+ *
+ *---------------------------------------------------------------------------
+ */
+#define ISO8601NOW(dest, now_gm) \
+strftime((dest), ISO8601_SIZE, "%Y%m%dT%H%M%SZ", (now_gm))
+
+/*---------------------------------------------------------------------------
+ *
+ * Macro: RFC7231NOW()
+ *
+ * Purpose:
+ *
+ * write "Day, dd Mmm YYYY HH:MM:SS GMT" to dest
+ * e.g., "Fri, 30 Jun 2017 20:41:55 GMT"
+ *
+ * wrapper for strftime()
+ *
+ * It is left to the programmer to check return value of
+ * RFC7231NOW (should equal RFC7231_SIZE - 1).
+ *
+ *---------------------------------------------------------------------------
+ */
+#define RFC7231NOW(dest, now_gm) \
+strftime((dest), RFC7231_SIZE, "%a, %d %b %Y %H:%M:%S GMT", (now_gm))
+
+
+/* Reasonable maximum length of a credential string.
+ * Provided for error-checking S3COMMS_FORMAT_CREDENTIAL (below).
+ * 17 <- "////aws4_request\0"
+ * 2 < "s3" (service)
+ * 8 <- "YYYYmmdd" (date)
+ * 128 <- (access_id)
+ * 155 :: sum
+ */
+#define S3COMMS_MAX_CREDENTIAL_SIZE 155
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Macro: H5FD_S3COMMS_FORMAT_CREDENTIAL()
+ *
+ * Purpose:
+ *
+ * Format "S3 Credential" string from inputs, for AWS4.
+ *
+ * Wrapper for HDsnprintf().
+ *
+ * _HAS NO ERROR-CHECKING FACILITIES_
+ * It is left to programmer to ensure that return value confers success.
+ * e.g.,
+ * ```
+ * assert( S3COMMS_MAX_CREDENTIAL_SIZE >=
+ * S3COMMS_FORMAT_CREDENTIAL(...) );
+ * ```
+ *
+ * "<access-id>/<date>/<aws-region>/<aws-service>/aws4_request"
+ * assuming that `dest` has adequate space.
+ *
+ * ALL inputs must be null-terminated strings.
+ *
+ * `access` should be the user's access key ID.
+ * `date` must be of format "YYYYmmdd".
+ * `region` should be relevant AWS region, i.e. "us-east-1".
+ * `service` should be "s3".
+ *
+ *---------------------------------------------------------------------------
+ */
+#define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service) \
+HDsnprintf((dest), S3COMMS_MAX_CREDENTIAL_SIZE, \
+ "%s/%s/%s/%s/aws4_request", \
+ (access), (iso8601_date), (region), (service))
+
+/*********************
+ * PUBLIC STRUCTURES *
+ *********************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: hrb_node_t
+ *
+ * HTTP Header Field Node
+ *
+ *
+ *
+ * Maintain a ordered (linked) list of HTTP Header fields.
+ *
+ * Provides efficient access and manipulation of a logical sequence of
+ * HTTP header fields, of particular use when composing an
+ * "S3 Canonical Request" for authentication.
+ *
+ * - The creation of a Canoncial Request involves:
+ * - convert field names to lower case
+ * - sort by this lower-case name
+ * - convert ": " name-value separator in HTTP string to ":"
+ * - get sorted lowercase names without field or separator
+ *
+ * As HTTP headers allow headers in any order (excepting the case of multiple
+ * headers with the same name), the list ordering can be optimized for Canonical
+ * Request creation, suggesting alphabtical order. For more expedient insertion
+ * and removal of elements in the list, linked list seems preferable to a
+ * dynamically-expanding array. The usually-smaller number of entries (5 or
+ * fewer) makes performance overhead of traversing the list trivial.
+ *
+ * The above requirements of creating at Canonical Request suggests a reasonable
+ * trade-off of speed for space with the option to compute elements as needed
+ * or to have the various elements prepared and stored in the structure
+ * (e.g. name, value, lowername, concatenated name:value)
+ * The structure currently is implemented to pre-compute.
+ *
+ * At all times, the "first" node of the list should be the least,
+ * alphabetically. For all nodes, the `next` node should be either NULL or
+ * of greater alphabetical value.
+ *
+ * Each node contains its own header field information, plus a pointer to the
+ * next node.
+ *
+ * It is not allowed to have multiple nodes with the same _lowercase_ `name`s
+ * in the same list
+ * (i.e., name is case-insensitive for access and modification.)
+ *
+ * All data (`name`, `value`, `lowername`, and `cat`) are null-terminated
+ * strings allocated specifically for their node.
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * "unique" idenfier number for the structure type
+ *
+ * `name` (char *)
+ *
+ * Case-meaningful name of the HTTP field.
+ * Given case is how it is supplied to networking code.
+ * e.g., "Range"
+ *
+ * `lowername` (char *)
+ *
+ * Lowercase copy of name.
+ * e.g., "range"
+ *
+ * `value` (char *)
+ *
+ * Case-meaningful value of HTTP field.
+ * e.g., "bytes=0-9"
+ *
+ * `cat` (char *)
+ *
+ * Concatenated, null-terminated string of HTTP header line,
+ * as the field would appear in an HTTP request.
+ * e.g., "Range: bytes=0-9"
+ *
+ * `next` (hrb_node_t *)
+ *
+ * Pointers to next node in the list, or NULL sentinel as end of list.
+ * Next node must have a greater `lowername` as determined by strcmp().
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct hrb_node_t {
+ unsigned long magic;
+ char *name;
+ char *value;
+ char *cat;
+ char *lowername;
+ struct hrb_node_t *next;
+} hrb_node_t;
+#define S3COMMS_HRB_NODE_MAGIC 0x7F5757UL
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: hrb_t
+ *
+ * HTTP Request Buffer structure
+ *
+ *
+ *
+ * Logically represent an HTTP request
+ *
+ * GET /myplace/myfile.h5 HTTP/1.1
+ * Host: over.rainbow.oz
+ * Date: Fri, 01 Dec 2017 12:35:04 CST
+ *
+ * <body>
+ *
+ * ...with fast, efficient access to and modification of primary and field
+ * elements.
+ *
+ * Structure for building HTTP requests while hiding much of the string
+ * processing required "under the hood."
+ *
+ * Information about the request target -- the first line -- and the body text,
+ * if any, are managed directly with this structure. All header fields, e.g.,
+ * "Host" and "Date" above, are created with a linked list of `hrb_node_t` and
+ * included in the request by a pointer to the head of the list.
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * "Magic" number confirming that this is an hrb_t structure and
+ * what operations are valid for it.
+ *
+ * Must be S3COMMS_HRB_MAGIC to be valid.
+ *
+ * `body` (char *) :
+ *
+ * Pointer to start of HTTP body.
+ *
+ * Can be NULL, in which case it is treated as the empty string, "".
+ *
+ * `body_len` (size_t) :
+ *
+ * Number of bytes (characters) in `body`. 0 if empty or NULL `body`.
+ *
+ * `first_header` (hrb_node_t *) :
+ *
+ * Pointer to first SORTED header node, if any.
+ * It is left to the programmer to ensure that this node and associated
+ * list is destroyed when done.
+ *
+ * `resource` (char *) :
+ *
+ * Pointer to resource URL string, e.g., "/folder/page.xhtml".
+ *
+ * `verb` (char *) :
+ *
+ * Pointer to HTTP verb string, e.g., "GET".
+ *
+ * `version` (char *) :
+ *
+ * Pointer to HTTP version string, e.g., "HTTP/1.1".
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct {
+ unsigned long magic;
+ char *body;
+ size_t body_len;
+ hrb_node_t *first_header;
+ char *resource;
+ char *verb;
+ char *version;
+} hrb_t;
+#define S3COMMS_HRB_MAGIC 0x6DCC84UL
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: parsed_url_t
+ *
+ *
+ * Represent a URL with easily-accessed pointers to logical elements within.
+ * These elements (components) are stored as null-terminated strings (or just
+ * NULLs). These components should be allocated for the structure, making the
+ * data as safe as possible from modification. If a component is NULL, it is
+ * either implicit in or absent from the URL.
+ *
+ * "http://mybucket.s3.amazonaws.com:8080/somefile.h5?param=value&arg=value"
+ * ^--^ ^-----------------------^ ^--^ ^---------^ ^-------------------^
+ * Scheme Host Port Resource Query/-ies
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * Structure identification and validation identifier.
+ * Identifies as `parsed_url_t` type.
+ *
+ * `scheme` (char *)
+ *
+ * String representing which protocol is to be expected.
+ * _Must_ be present.
+ * "http", "https", "ftp", e.g.
+ *
+ * `host` (char *)
+ *
+ * String of host, either domain name, IPv4, or IPv6 format.
+ * _Must_ be present.
+ * "over.rainbow.oz", "192.168.0.1", "[0000:0000:0000:0001]"
+ *
+ * `port` (char *)
+ *
+ * String representation of specified port. Must resolve to a valid unsigned
+ * integer.
+ * "9000", "80"
+ *
+ * `path` (char *)
+ *
+ * Path to resource on host. If not specified, assumes root "/".
+ * "lollipop_guild.wav", "characters/witches/white.dat"
+ *
+ * `query` (char *)
+ *
+ * Single string of all query parameters in url (if any).
+ * "arg1=value1&arg2=value2"
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct {
+ unsigned long magic;
+ char *scheme; /* required */
+ char *host; /* required */
+ char *port;
+ char *path;
+ char *query;
+} parsed_url_t;
+#define S3COMMS_PARSED_URL_MAGIC 0x21D0DFUL
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: s3r_t
+ *
+ *
+ *
+ * S3 request structure "handle".
+ *
+ * Holds persistent information for Amazon S3 requests.
+ *
+ * Instantiated through `H5FD_s3comms_s3r_open()`, copies data into self.
+ *
+ * Intended to be re-used for operations on a remote object.
+ *
+ * Cleaned up through `H5FD_s3comms_s3r_close()`.
+ *
+ * _DO NOT_ share handle between threads: curl easy handle `curlhandle` has
+ * undefined behavior if called to perform in multiple threads.
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * "magic" number identifying this structure as unique type.
+ * MUST equal `S3R_MAGIC` to be valid.
+ *
+ * `curlhandle` (CURL)
+ *
+ * Pointer to the curl_easy handle generated for the request.
+ *
+ * `httpverb` (char *)
+ *
+ * Pointer to NULL-terminated string. HTTP verb,
+ * e.g. "GET", "HEAD", "PUT", etc.
+ *
+ * Default is NULL, resulting in a "GET" request.
+ *
+ * `purl` (parsed_url_t *)
+ *
+ * Pointer to structure holding the elements of URL for file open.
+ *
+ * e.g., "http://bucket.aws.com:8080/myfile.dat?q1=v1&q2=v2"
+ * parsed into...
+ * { scheme: "http"
+ * host: "bucket.aws.com"
+ * port: "8080"
+ * path: "myfile.dat"
+ * query: "q1=v1&q2=v2"
+ * }
+ *
+ * Cannot be NULL.
+ *
+ * `region` (char *)
+ *
+ * Pointer to NULL-terminated string, specifying S3 "region",
+ * e.g., "us-east-1".
+ *
+ * Required to authenticate.
+ *
+ * `secret_id` (char *)
+ *
+ * Pointer to NULL-terminated string for "secret" access id to S3 resource.
+ *
+ * Requred to authenticate.
+ *
+ * `signing_key` (unsigned char *)
+ *
+ * Pointer to `SHA256_DIGEST_LENGTH`-long string for "re-usable" signing
+ * key, generated via
+ * `HMAC-SHA256(HMAC-SHA256(HMAC-SHA256(HMAC-SHA256("AWS4<secret_key>",
+ * "<yyyyMMDD"), "<aws-region>"), "<aws-service>"), "aws4_request")`
+ * which may be re-used for several (up to seven (7)) days from creation?
+ * Computed once upon file open.
+ *
+ * Requred to authenticate.
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct {
+ unsigned long magic;
+ CURL *curlhandle;
+ size_t filesize;
+ char *httpverb;
+ parsed_url_t *purl;
+ char *region;
+ char *secret_id;
+ unsigned char *signing_key;
+} s3r_t;
+
+#define S3COMMS_S3R_MAGIC 0x44d8d79
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*******************************************
+ * DECLARATION OF HTTP FIELD LIST ROUTINES *
+ *******************************************/
+
+H5_DLL herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L,
+ const char *name,
+ const char *value);
+
+/***********************************************
+ * DECLARATION OF HTTP REQUEST BUFFER ROUTINES *
+ ***********************************************/
+
+H5_DLL herr_t H5FD_s3comms_hrb_destroy(hrb_t **buf);
+
+H5_DLL hrb_t * H5FD_s3comms_hrb_init_request(const char *verb,
+ const char *resource,
+ const char *host);
+
+/*************************************
+ * DECLARATION OF S3REQUEST ROUTINES *
+ *************************************/
+
+H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle);
+
+H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle);
+
+H5_DLL s3r_t * H5FD_s3comms_s3r_open(const char url[],
+ const char region[],
+ const char id[],
+ const unsigned char signing_key[]);
+
+H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle,
+ haddr_t offset,
+ size_t len,
+ void *dest);
+
+/*********************************
+ * DECLARATION OF OTHER ROUTINES *
+ *********************************/
+
+H5_DLL struct tm * gmnow(void);
+
+H5_DLL herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest,
+ int cr_size,
+ char *signed_headers_dest,
+ int sh_size,
+ hrb_t *http_request);
+
+H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest,
+ const unsigned char *msg,
+ size_t msg_len,
+ hbool_t lowercase);
+
+H5_DLL herr_t H5FD_s3comms_free_purl(parsed_url_t *purl);
+
+H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key,
+ size_t key_len,
+ const char *msg,
+ size_t msg_len,
+ char *dest);
+
+H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name,
+ char *key_id_out,
+ char *secret_access_key_out,
+ char *aws_region_out);
+
+H5_DLL herr_t H5FD_s3comms_nlowercase(char *dest,
+ const char *s,
+ size_t len);
+
+H5_DLL herr_t H5FD_s3comms_parse_url(const char *str,
+ parsed_url_t **purl);
+
+H5_DLL herr_t H5FD_s3comms_percent_encode_char(char *repr,
+ const unsigned char c,
+ size_t *repr_len);
+
+H5_DLL herr_t H5FD_s3comms_signing_key(unsigned char *md,
+ const char *secret,
+ const char *region,
+ const char *iso8601now);
+
+H5_DLL herr_t H5FD_s3comms_tostringtosign(char *dest,
+ const char *req_str,
+ const char *now,
+ const char *region);
+
+H5_DLL herr_t H5FD_s3comms_trim(char *dest,
+ char *s,
+ size_t s_len,
+ size_t *n_written);
+
+H5_DLL herr_t H5FD_s3comms_uriencode(char *dest, const char *s, size_t s_len,
+ hbool_t encode_slash, size_t *n_written);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+#endif /* ifndef H5FDs3comms_H */
+
diff --git a/src/Makefile.am b/src/Makefile.am
index 93a90be..4774ae2 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -53,7 +53,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Fmount.c H5Fmpi.c H5Fquery.c \
H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FD.c H5FDcore.c \
- H5FDdirect.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \
+ H5FDdirect.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \
H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSsection.c H5FSstat.c H5FStest.c \
H5G.c H5Gbtree2.c H5Gcache.c \
@@ -98,9 +98,13 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Topaque.c \
H5Torder.c \
H5Tpad.c H5Tprecis.c H5Tstrpad.c H5Tvisit.c H5Tvlen.c H5TS.c H5VM.c H5WB.c H5Z.c \
- H5Zdeflate.c H5Zfletcher32.c H5Znbit.c H5Zshuffle.c H5Zszip.c \
- H5Zscaleoffset.c H5Ztrans.c
+ H5Zdeflate.c H5Zfletcher32.c H5Znbit.c H5Zshuffle.c \
+ H5Zscaleoffset.c H5Zszip.c H5Ztrans.c
+# Only compile the read-only S3 VFD if necessary
+if ROS3_VFD_CONDITIONAL
+ libhdf5_la_SOURCES += H5FDros3.c H5FDs3comms.c
+endif
# Public headers
include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5version.h \
@@ -108,8 +112,8 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers
H5Cpublic.h H5Dpublic.h \
H5Epubgen.h H5Epublic.h H5Fpublic.h \
H5FDpublic.h H5FDcore.h H5FDdirect.h \
- H5FDfamily.h H5FDlog.h H5FDmpi.h H5FDmpio.h \
- H5FDmulti.h H5FDsec2.h H5FDstdio.h \
+ H5FDfamily.h H5FDhdfs.h H5FDlog.h H5FDmpi.h H5FDmpio.h \
+ H5FDmulti.h H5FDros3.h H5FDsec2.h H5FDstdio.h \
H5Gpublic.h H5Ipublic.h H5Lpublic.h \
H5MMpublic.h H5Opublic.h H5Ppublic.h \
H5PLextern.h H5PLpublic.h \
diff --git a/src/Makefile.in b/src/Makefile.in
index 97bfbc7..8284041 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -163,8 +163,9 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \
H5Faccum.lo H5Fcwfs.lo H5Fdbg.lo H5Fefc.lo H5Ffake.lo H5Fio.lo \
H5Fmount.lo H5Fmpi.lo H5Fquery.lo H5Fsfile.lo H5Fsuper.lo \
H5Fsuper_cache.lo H5Ftest.lo H5FD.lo H5FDcore.lo H5FDdirect.lo \
- H5FDfamily.lo H5FDint.lo H5FDlog.lo H5FDmpi.lo H5FDmpio.lo \
- H5FDmulti.lo H5FDsec2.lo H5FDspace.lo H5FDstdio.lo H5FL.lo \
+ H5FDfamily.lo H5FDhdfs.lo H5FDint.lo H5FDlog.lo H5FDs3comms.lo \
+ H5FDmpi.lo H5FDmpio.lo H5FDmulti.lo H5FDros3.lo H5FDsec2.lo \
+ H5FDspace.lo H5FDstdio.lo H5FL.lo \
H5FO.lo H5FS.lo H5FScache.lo H5FSdbg.lo H5FSsection.lo \
H5FSstat.lo H5FStest.lo H5G.lo H5Gbtree2.lo H5Gcache.lo \
H5Gcompact.lo H5Gdense.lo H5Gdeprec.lo H5Gent.lo H5Gint.lo \
@@ -530,6 +531,7 @@ H5_CPPFLAGS = @H5_CPPFLAGS@
H5_CXXFLAGS = @H5_CXXFLAGS@
H5_FCFLAGS = @H5_FCFLAGS@
H5_FORTRAN_SHARED = @H5_FORTRAN_SHARED@
+HAVE_LIBHDFS = @HAVE_LIBHDFS@
H5_LDFLAGS = @H5_LDFLAGS@
H5_VERSION = @H5_VERSION@
HADDR_T = @HADDR_T@
@@ -590,6 +592,7 @@ PATH_SEPARATOR = @PATH_SEPARATOR@
PERL = @PERL@
RANLIB = @RANLIB@
ROOT = @ROOT@
+ROS3_VFD = @ROS3_VFD@
RUNPARALLEL = @RUNPARALLEL@
RUNSERIAL = @RUNSERIAL@
R_INTEGER = @R_INTEGER@
@@ -781,8 +784,8 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Fmount.c H5Fmpi.c H5Fquery.c \
H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FD.c H5FDcore.c \
- H5FDdirect.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \
- H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c \
+ H5FDdirect.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c H5FDs3comms.c \
+ H5FDmulti.c H5FDros3.c H5FDsec2.c H5FDspace.c H5FDstdio.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSsection.c H5FSstat.c H5FStest.c \
H5G.c H5Gbtree2.c H5Gcache.c \
H5Gcompact.c H5Gdense.c H5Gdeprec.c H5Gent.c \
diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in
index 5cb0930..5b65d77 100644
--- a/src/libhdf5.settings.in
+++ b/src/libhdf5.settings.in
@@ -3,37 +3,47 @@
General Information:
-------------------
- HDF5 Version: @H5_VERSION@
- Configured on: @CONFIG_DATE@
- Configured by: @CONFIG_USER@
- Configure mode: @CONFIG_MODE@
- Host system: @host_cpu@-@host_vendor@-@host_os@
- Uname information: @UNAME_INFO@
- Byte sex: @BYTESEX@
- Libraries: @STATIC_SHARED@
- Installation point: @prefix@
+ HDF5 Version: @H5_VERSION@
+ Configured on: @CONFIG_DATE@
+ Configured by: @CONFIG_USER@
+ Configure mode: @CONFIG_MODE@
+ Host system: @host_cpu@-@host_vendor@-@host_os@
+ Uname information: @UNAME_INFO@
+ Byte sex: @BYTESEX@
+ Installation point: @prefix@
Compiling Options:
------------------
- Compilation Mode: @CONFIG_MODE@
+ Build Mode: @BUILD_MODE@
+ Debugging Symbols: @SYMBOLS@
+ Asserts: @ASSERTS@
+ Profiling: @PROFILING@
+ Optimization Level: @OPTIMIZATION@
+
+Linking Options:
+----------------
+ Libraries: @STATIC_SHARED@
+ Statically Linked Executables: @LT_STATIC_EXEC@
+ LDFLAGS: @LDFLAGS@
+ H5_LDFLAGS: @H5_LDFLAGS@
+ AM_LDFLAGS: @AM_LDFLAGS@
+ Extra libraries: @LIBS@
+ Archiver: @AR@
+ AR_FLAGS: @AR_FLAGS@
+ Ranlib: @RANLIB@
+
+Languages:
+----------
+ C: yes
C Compiler: @CC_VERSION@
- CFLAGS: @CFLAGS@
- H5_CFLAGS: @H5_CFLAGS@
- AM_CFLAGS: @AM_CFLAGS@
CPPFLAGS: @CPPFLAGS@
H5_CPPFLAGS: @H5_CPPFLAGS@
AM_CPPFLAGS: @AM_CPPFLAGS@
+ C Flags: @CFLAGS@
+ H5 C Flags: @H5_CFLAGS@
+ AM C Flags: @AM_CFLAGS@
Shared C Library: @enable_shared@
Static C Library: @enable_static@
- Statically Linked Executables: @STATIC_EXEC@
- LDFLAGS: @LDFLAGS@
- H5_LDFLAGS: @H5_LDFLAGS@
- AM_LDFLAGS: @AM_LDFLAGS@
- Extra libraries: @LIBS@
- Archiver: @AR@
- Ranlib: @RANLIB@
- Debugged Packages: @DEBUG_PKG@
- API Tracing: @TRACE_API@
Languages:
----------
@@ -56,19 +66,21 @@ Languages:
Features:
---------
- Parallel HDF5: @PARALLEL@
- High Level library: @HDF5_HL@
- Build HDF5 Tests: @HDF5_TESTS@
- Build HDF5 Tools: @HDF5_TOOLS@
- Threadsafety: @THREADSAFE@
- Default API Mapping: @DEFAULT_API_VERSION@
- With Deprecated Public Symbols: @DEPRECATED_SYMBOLS@
- I/O filters (external): @EXTERNAL_FILTERS@
- MPE: @MPE@
- Direct VFD: @DIRECT_VFD@
- dmalloc: @HAVE_DMALLOC@
-Clear file buffers before write: @CLEARFILEBUF@
- Using memory checker: @USINGMEMCHECKER@
- Function Stack Tracing: @CODESTACK@
- Strict File Format Checks: @STRICT_FORMAT_CHECKS@
- Optimization Instrumentation: @INSTRUMENT@
+ Parallel HDF5: @PARALLEL@
+ High-level library: @HDF5_HL@
+ Build HDF5 Tests: @HDF5_TESTS@
+ Build HDF5 Tools: @HDF5_TOOLS@
+ Threadsafety: @THREADSAFE@
+ Default API mapping: @DEFAULT_API_VERSION@
+ With deprecated public symbols: @DEPRECATED_SYMBOLS@
+ I/O filters (external): @EXTERNAL_FILTERS@
+ MPE: @MPE@
+ Direct VFD: @DIRECT_VFD@
+ (Read-Only) S3 VFD: @ROS3_VFD@
+ (Read-Only) HDFS VFD: @HAVE_LIBHDFS@
+ dmalloc: @HAVE_DMALLOC@
+ Clear file buffers before write: @CLEARFILEBUF@
+ Using memory checker: @USINGMEMCHECKER@
+ Function stack tracing: @CODESTACK@
+ Strict file format checks: @STRICT_FORMAT_CHECKS@
+ Optimization instrumentation: @INSTRUMENT@
generated by cgit v0.12 at 2024-07-07 22:46:58 (GMT)