Prevent buffer overrun in H5S_select_deserialize (#2931)

* Prevent buffer overrun in H5S_select_deserialize

The call to H5S_select_deserialize from H5S_decode doesn't have
the buffer size available to it, so to allow decoding there
I set it to assume a max size buffer for now.

Making the buffer size known in H5S_decode could be done by
modifying the external API's H5Sdecode, or splitting H5Sdecode
into two functions using a macro (similar to H5Sencode), with the
macro taking one argument and assuming a max buffer size.

* Conditional buffer check in H5S_select_deserialize

Moved and renamed a macro for only checking buffer overflow when
buffer size is known from H5Odtype.c to H5private.h,
so it can be used throughout the library.

Also silenced some build warnings about types.

1 files changed, 2 insertions, 2 deletions

diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index 4303eee..ac46d9e 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -190,7 +190,7 @@ typedef struct H5S_sel_iter_op_t {
 #define H5S_SELECT_SHAPE_SAME(S1, S2)             (H5S_select_shape_same(S1, S2))
 #define H5S_SELECT_INTERSECT_BLOCK(S, START, END) (H5S_select_intersect_block(S, START, END))
 #define H5S_SELECT_RELEASE(S)                     (H5S_select_release(S))
-#define H5S_SELECT_DESERIALIZE(S, BUF)            (H5S_select_deserialize(S, BUF))
+#define H5S_SELECT_DESERIALIZE(S, BUF, BUF_SIZE)  (H5S_select_deserialize(S, BUF, BUF_SIZE))
 
 /* Forward declaration of structs used below */
 struct H5O_t;
@@ -229,7 +229,7 @@ H5_DLL htri_t  H5S_extent_equal(const H5S_t *ds1, const H5S_t *ds2);
 H5_DLL herr_t  H5S_extent_copy(H5S_t *dst, const H5S_t *src);
 
 /* Operations on selections */
-H5_DLL herr_t       H5S_select_deserialize(H5S_t **space, const uint8_t **p);
+H5_DLL herr_t       H5S_select_deserialize(H5S_t **space, const uint8_t **p, const size_t p_size);
 H5_DLL H5S_sel_type H5S_get_select_type(const H5S_t *space);
 H5_DLL herr_t   H5S_select_iterate(void *buf, const H5T_t *type, H5S_t *space, const H5S_sel_iter_op_t *op,
                                    void *op_data);