From 2ece0d83809849e68e9c194ff3d340982b527256 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 5 Apr 2019 12:54:13 -0700 Subject: created LZ4_initStream() - promoted LZ4_resetStream_fast() to stable - moved LZ4_resetStream() into deprecate, but without triggering a compiler warning - update all sources to no longer rely on LZ4_resetStream() note : LZ4_initStream() proposal is slightly different : it's able to initialize any buffer, provided that it's large enough. To this end, it accepts a void*, and returns an LZ4_stream_t*. --- doc/lz4_manual.html | 87 +++++++++++++++++---------------- examples/blockStreaming_doubleBuffer.c | 2 +- examples/dictionaryRandomAccess.c | 4 +- lib/lz4.c | 17 +++++-- lib/lz4.h | 88 +++++++++++++++++----------------- lib/lz4frame.c | 4 +- lib/lz4hc.h | 4 +- tests/fullbench.c | 4 +- tests/fuzzer.c | 6 +-- 9 files changed, 111 insertions(+), 105 deletions(-) diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html index b738c8d..5db3ec9 100644 --- a/doc/lz4_manual.html +++ b/doc/lz4_manual.html @@ -155,9 +155,23 @@ int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int src

Streaming Compression Functions


 
-
void LZ4_resetStream (LZ4_stream_t* streamPtr);
-
  An LZ4_stream_t structure can be allocated once and re-used multiple times.
-  Use this function to start compressing a new stream.
+
void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
+
  Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
+  (e.g., LZ4_compress_fast_continue()).
+
+  An LZ4_stream_t must be initialized once.
+  This is automatically done when created by LZ4_createStream().
+  However, should the LZ4_stream_t be simply declared on stack (for example),
+  it's necessary to initialize first using LZ4_initStream().
+
+  After that, start any new stream with LZ4_resetStream_fast().
+  A same LZ4_stream_t can be re-used multiple times consecutively
+  and compress multiple streams,
+  provided that it starts each new stream with LZ4_resetStream_fast().
+
+  LZ4_resetStream_fast() is much faster than LZ4_initStream(),
+  but is not compatible with memory regions containing garbage data.
+  For this reason, LZ4_stream_t must be initialized at least once,
  
 


 
@@ -290,43 +304,6 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
 

 
-
LZ4LIB_STATIC_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
-
  Use this to prepare a context for a new chain of calls to a streaming API
-  (e.g., LZ4_compress_fast_continue()).
-
-  Note:
-  To stay on the safe side, when LZ4_stream_t is used for the first time,
-  it should be either created using LZ4_createStream() or
-  initialized using LZ4_resetStream().
-
-  Note:
-  Using this in advance of a non-streaming-compression function is redundant,
-  since they all perform their own custom reset internally.
-
-  Differences from LZ4_resetStream():
-  When an LZ4_stream_t is known to be in an internally coherent state,
-  it will be prepared for a new compression with almost no work.
-  Otherwise, it will fall back to the full, expensive reset.
-
-  LZ4_streams are guaranteed to be in a valid state when:
-  - returned from LZ4_createStream()
-  - reset by LZ4_resetStream()
-  - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged
-  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
-  - the stream was in a valid state and was then used in any compression call
-    that returned success
-  - the stream was in an indeterminate state and was used in a compression
-    call that fully reset the state (e.g., LZ4_compress_fast_extState()) and
-    that returned success
-
-  Note:
-  A stream that was used in a compression call that did not return success
-  (e.g., LZ4_compress_fast_continue()), can still be passed to this function,
-  however, it's history is not preserved because of previous compression
-  failure.
- 
-


-
 
LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
  A variant of LZ4_compress_fast_extState().
 
@@ -394,10 +371,24 @@ union LZ4_stream_u {
     LZ4_stream_t_internal internal_donotuse;
 } ;  /* previously typedef'd to LZ4_stream_t */
 
  information structure to track an LZ4 stream.
-  init this structure with LZ4_resetStream() before first use.
-  note : only use in association with static linking !
-         this definition is not API/ABI safe,
-         it may change in a future version !
+  LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
+  The structure definition can be convenient for static allocation
+  (on stack, or as part of larger structure).
+  Init this structure with LZ4_initStream() before first use.
+  note : only use this definition in association with static linking !
+    this definition is not API/ABI safe, and may change in a future version.
+ 
+


+
+
LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
+
  An LZ4_stream_t structure must be initialized at least once.
+  While this is automatically done when invoking LZ4_createStream(),
+  it's not when the structure is simply declared on stack (for example).
+  Use this function to properly initialize a newly declared LZ4_stream_t.
+  It can also accept any arbitrary buffer of sufficient size as input,
+  and will return a pointer of proper type upon initialization.
+  Note : initialization can fail if size < sizeof(LZ4_stream_t).
+  In which case, the function will @return NULL.
  
 


 
@@ -477,5 +468,13 @@ int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize,
  
 


 
+
//LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
+
  An LZ4_stream_t structure must be initialized at least once.
+  This is done with LZ4_initStream(), or LZ4_resetStream().
+  Consider switching to LZ4_initStream(),
+  invoking LZ4_resetStream() will trigger deprecation warnings in the future.
+ 
+


+
 
 
diff --git a/examples/blockStreaming_doubleBuffer.c b/examples/blockStreaming_doubleBuffer.c
index acb3455..3f719d3 100644
--- a/examples/blockStreaming_doubleBuffer.c
+++ b/examples/blockStreaming_doubleBuffer.c
@@ -44,7 +44,7 @@ void test_compress(FILE* outFp, FILE* inpFp)
     char inpBuf[2][BLOCK_BYTES];
     int  inpBufIndex = 0;
 
-    LZ4_resetStream(lz4Stream);
+    LZ4_initStream(lz4Stream, sizeof (*lz4Stream));
 
     for(;;) {
         char* const inpPtr = inpBuf[inpBufIndex];
diff --git a/examples/dictionaryRandomAccess.c b/examples/dictionaryRandomAccess.c
index 291fd08..ecb3b2d 100644
--- a/examples/dictionaryRandomAccess.c
+++ b/examples/dictionaryRandomAccess.c
@@ -11,7 +11,7 @@
 #include 
 #include 
 
-#define MIN(x, y) (x) < (y) ? (x) : (y)
+#define MIN(x, y)  ((x) < (y) ? (x) : (y))
 
 enum {
     BLOCK_BYTES = 1024,  /* 1 KiB of uncompressed data in a block */
@@ -63,7 +63,7 @@ void test_compress(FILE* outFp, FILE* inpFp, void *dict, int dictSize)
     int *offsetsEnd = offsets;
 
 
-    LZ4_resetStream(lz4Stream);
+    LZ4_initStream(lz4Stream, sizeof(*lz4Stream));
 
     /* Write header magic */
     write_bin(outFp, kTestMagic, sizeof(kTestMagic));
diff --git a/lib/lz4.c b/lib/lz4.c
index df92162..e0fcf0f 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1125,9 +1125,8 @@ _failure:
 
 int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
 {
-    LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse;
+    LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse;
     if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
-    LZ4_resetStream((LZ4_stream_t*)state);
     if (maxOutputSize >= LZ4_compressBound(inputSize)) {
         if (inputSize < LZ4_64Klimit) {
             return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
@@ -1221,7 +1220,7 @@ int LZ4_compress_default(const char* src, char* dst, int srcSize, int maxOutputS
 int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapacity, int acceleration)
 {
     LZ4_stream_t ctx;
-    LZ4_resetStream(&ctx);
+    LZ4_initStream(&ctx, sizeof(ctx));
 
     if (srcSize < LZ4_64Klimit) {
         return LZ4_compress_generic(&ctx.internal_donotuse, src, dst, srcSize, NULL, dstCapacity, limitedOutput, byU16,    noDict, noDictIssue, acceleration);
@@ -1237,7 +1236,7 @@ int LZ4_compress_fast_force(const char* src, char* dst, int srcSize, int dstCapa
  * _continue() call without resetting it. */
 static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
 {
-    LZ4_resetStream(state);
+    LZ4_initStream(state, sizeof (*state));
 
     if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) {  /* compression success is guaranteed */
         return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
@@ -1281,10 +1280,18 @@ LZ4_stream_t* LZ4_createStream(void)
     LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal));    /* A compilation error here means LZ4_STREAMSIZE is not large enough */
     DEBUGLOG(4, "LZ4_createStream %p", lz4s);
     if (lz4s == NULL) return NULL;
-    LZ4_resetStream(lz4s);
+    LZ4_initStream(lz4s, sizeof(*lz4s));
     return lz4s;
 }
 
+LZ4_stream_t* LZ4_initStream (void* buffer, size_t size)
+{
+    DEBUGLOG(5, "LZ4_initStream");
+    if (size < sizeof(LZ4_stream_t)) return NULL;
+    MEM_INIT(buffer, 0, sizeof(LZ4_stream_t));
+    return (LZ4_stream_t*)buffer;
+}
+
 void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
 {
     DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream);
diff --git a/lib/lz4.h b/lib/lz4.h
index 1dc8e00..3fc67bb 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -238,11 +238,25 @@ typedef union LZ4_stream_u LZ4_stream_t;  /* incomplete type (defined later) */
 LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
 LZ4LIB_API int           LZ4_freeStream (LZ4_stream_t* streamPtr);
 
-/*! LZ4_resetStream() :
- *  An LZ4_stream_t structure can be allocated once and re-used multiple times.
- *  Use this function to start compressing a new stream.
+/*! LZ4_resetStream_fast() : v1.9.0+
+ *  Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
+ *  (e.g., LZ4_compress_fast_continue()).
+ *
+ *  An LZ4_stream_t must be initialized once.
+ *  This is automatically done when created by LZ4_createStream().
+ *  However, should the LZ4_stream_t be simply declared on stack (for example),
+ *  it's necessary to initialize first using LZ4_initStream().
+ *
+ *  After that, start any new stream with LZ4_resetStream_fast().
+ *  A same LZ4_stream_t can be re-used multiple times consecutively
+ *  and compress multiple streams,
+ *  provided that it starts each new stream with LZ4_resetStream_fast().
+ *
+ *  LZ4_resetStream_fast() is much faster than LZ4_initStream(),
+ *  but is not compatible with memory regions containing garbage data.
+ *  For this reason, LZ4_stream_t must be initialized at least once,
  */
-LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
+LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
 
 /*! LZ4_loadDict() :
  *  Use this function to load a static dictionary into LZ4_stream_t.
@@ -394,42 +408,6 @@ LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int sr
 
 #ifdef LZ4_STATIC_LINKING_ONLY
 
-/*! LZ4_resetStream_fast() :
- *  Use this to prepare a context for a new chain of calls to a streaming API
- *  (e.g., LZ4_compress_fast_continue()).
- *
- *  Note:
- *  To stay on the safe side, when LZ4_stream_t is used for the first time,
- *  it should be either created using LZ4_createStream() or
- *  initialized using LZ4_resetStream().
- *
- *  Note:
- *  Using this in advance of a non-streaming-compression function is redundant,
- *  since they all perform their own custom reset internally.
- *
- *  Differences from LZ4_resetStream():
- *  When an LZ4_stream_t is known to be in an internally coherent state,
- *  it will be prepared for a new compression with almost no work.
- *  Otherwise, it will fall back to the full, expensive reset.
- *
- *  LZ4_streams are guaranteed to be in a valid state when:
- *  - returned from LZ4_createStream()
- *  - reset by LZ4_resetStream()
- *  - memset(stream, 0, sizeof(LZ4_stream_t)), though this is discouraged
- *  - the stream was in a valid state and was reset by LZ4_resetStream_fast()
- *  - the stream was in a valid state and was then used in any compression call
- *    that returned success
- *  - the stream was in an indeterminate state and was used in a compression
- *    call that fully reset the state (e.g., LZ4_compress_fast_extState()) and
- *    that returned success
- *
- *  Note:
- *  A stream that was used in a compression call that did not return success
- *  (e.g., LZ4_compress_fast_continue()), can still be passed to this function,
- *  however, it's history is not preserved because of previous compression
- *  failure.
- */
-LZ4LIB_STATIC_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
 
 /*! LZ4_compress_fast_extState_fastReset() :
  *  A variant of LZ4_compress_fast_extState().
@@ -530,10 +508,12 @@ typedef struct {
 
 /*! LZ4_stream_t :
  *  information structure to track an LZ4 stream.
- *  init this structure with LZ4_resetStream() before first use.
- *  note : only use in association with static linking !
- *         this definition is not API/ABI safe,
- *         it may change in a future version !
+ *  LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
+ *  The structure definition can be convenient for static allocation
+ *  (on stack, or as part of larger structure).
+ *  Init this structure with LZ4_initStream() before first use.
+ *  note : only use this definition in association with static linking !
+ *    this definition is not API/ABI safe, and may change in a future version.
  */
 #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ )
 #define LZ4_STREAMSIZE     (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
@@ -542,6 +522,18 @@ union LZ4_stream_u {
     LZ4_stream_t_internal internal_donotuse;
 } ;  /* previously typedef'd to LZ4_stream_t */
 
+/*! LZ4_initStream() :
+ *  An LZ4_stream_t structure must be initialized at least once.
+ *  While this is automatically done when invoking LZ4_createStream(),
+ *  it's not when the structure is simply declared on stack (for example).
+ *  Use this function to properly initialize a newly declared LZ4_stream_t.
+ *  It can also accept any arbitrary buffer of sufficient size as input,
+ *  and will return a pointer of proper type upon initialization.
+ *  Note : initialization can fail if size < sizeof(LZ4_stream_t).
+ *  In which case, the function will @return NULL.
+ */
+LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
+
 
 /*! LZ4_streamDecode_t :
  *  information structure to track an LZ4 stream during decompression.
@@ -652,6 +644,14 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch
 LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") LZ4LIB_API
 int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
 
+/*! LZ4_resetStream() :
+ *  An LZ4_stream_t structure must be initialized at least once.
+ *  This is done with LZ4_initStream(), or LZ4_resetStream().
+ *  Consider switching to LZ4_initStream(),
+ *  invoking LZ4_resetStream() will trigger deprecation warnings in the future.
+ */
+LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
+
 
 #endif /* LZ4_H_2983827168210 */
 
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index f076204..3b4fcee 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -446,7 +446,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
     if (preferencesPtr == NULL ||
         preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
     {
-        LZ4_resetStream(&lz4ctx);
+        LZ4_initStream(&lz4ctx, sizeof(lz4ctx));
         cctxPtr->lz4CtxPtr = &lz4ctx;
         cctxPtr->lz4CtxAlloc = 1;
         cctxPtr->lz4CtxState = 1;
@@ -628,7 +628,7 @@ size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
             /* otherwise, a sufficient buffer is allocated, but we need to
              * reset it to the correct context type */
             if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
-                LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
+                LZ4_initStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr, sizeof (LZ4_stream_t));
             } else {
                 LZ4_initStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
             }
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 91b14ee..fbf4518 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -152,7 +152,7 @@ LZ4LIB_API int             LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
   just by resetting it, using LZ4_resetStreamHC_fast().
 */
 
-LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel);
+LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel);   /* v1.9.0+ */
 LZ4LIB_API int  LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
 
 LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr,
@@ -265,7 +265,7 @@ union LZ4_streamHC_u {
  * Static allocation shall only be used in combination with static linking.
  */
 
-LZ4LIB_API void LZ4_initStreamHC (void* streamHCPtr, int compressionLevel);
+LZ4LIB_API void LZ4_initStreamHC (void* streamHCPtr, int compressionLevel);  /* v1.9.0+ */
 
 
 /*-************************************
diff --git a/tests/fullbench.c b/tests/fullbench.c
index faa50cb..61d6b78 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -161,12 +161,12 @@ static size_t BMK_findMaxMem(U64 requiredMem)
 static LZ4_stream_t LZ4_stream;
 static void local_LZ4_resetDictT(void)
 {
-    LZ4_resetStream(&LZ4_stream);
+    LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
 }
 
 static void local_LZ4_createStream(void)
 {
-    LZ4_resetStream(&LZ4_stream);
+    LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
 }
 
 static int local_LZ4_saveDict(const char* in, char* out, int inSize)
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 1efe322..d899cef 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -641,7 +641,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
         /* Compress using dictionary */
         FUZ_DISPLAYTEST("test LZ4_compress_fast_continue() with dictionary of size %i", dictSize);
         {   LZ4_stream_t LZ4_stream;
-            LZ4_resetStream(&LZ4_stream);
+            LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
             LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1);   /* Just to fill hash tables */
             blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
@@ -742,7 +742,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
             FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary()");
             LZ4_loadDict(&LZ4dict, dict, dictSize);
-            LZ4_resetStream(&LZ4_stream);
+            LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dict);
             blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
             FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue using extDictCtx failed");
@@ -1000,7 +1000,7 @@ static void FUZ_unitTests(int compressionLevel)
 
         /* simple compression test */
         crcOrig = XXH64(testInput, testCompressedSize, 0);
-        LZ4_resetStream(&streamingState);
+        LZ4_initStream(&streamingState, sizeof(streamingState));
         result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1);
         FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!");
         FUZ_CHECKTEST(streamingState.internal_donotuse.dirty, "context should be clean")
-- 
cgit v0.12