From 4aeb5020c35e2464c25eb69d8bf6c7645b8faf20 Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Fri, 3 Jun 2022 15:38:54 +0200
Subject: frame-api: add method to insert uncomressed data

new method `uncompressed_update` allows to insert blocks without
compression into the lz4 stream.
The usage is documented in the frameCompress example

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 doc/lz4frame_manual.html |   5 ++
 examples/frameCompress.c | 133 ++++++++++++++++++++++-----
 lib/lz4.c                |  14 ++-
 lib/lz4.h                |   2 +
 lib/lz4frame.c           | 228 ++++++++++++++++++++++++++++++++++-------------
 lib/lz4frame.h           |  23 +++++
 lib/lz4hc.c              |  17 ++--
 lib/lz4hc.h              |   2 +
 8 files changed, 329 insertions(+), 95 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 0040c98..9490b51 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -75,6 +75,11 @@
     LZ4F_OBSOLETE_ENUM(skippableFrame)
 } LZ4F_frameType_t;
 </b></pre><BR>
+<pre><b>typedef enum {
+  LZ4B_COMPRESSED,
+  LZ4B_UNCOMPRESSED,
+} LZ4F_blockCompression_t;
+</b></pre><BR>
 <pre><b>typedef struct {
   LZ4F_blockSizeID_t     blockSizeID;         </b>/* max64KB, max256KB, max1MB, max4MB; 0 == default */<b>
   LZ4F_blockMode_t       blockMode;           </b>/* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */<b>
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 9eaa4da..3219014 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -11,9 +11,9 @@
 #include <errno.h>
 #include <assert.h>
 
+#include <getopt.h>
 #include <lz4frame.h>
 
-
 #define IN_CHUNK_SIZE  (16*1024)
 
 static const LZ4F_preferences_t kPrefs = {
@@ -57,10 +57,11 @@ static compressResult_t
 compress_file_internal(FILE* f_in, FILE* f_out,
                        LZ4F_compressionContext_t ctx,
                        void* inBuff,  size_t inChunkSize,
-                       void* outBuff, size_t outCapacity)
+                       void* outBuff, size_t outCapacity,
+                       FILE* f_unc, long uncOffset)
 {
     compressResult_t result = { 1, 0, 0 };  /* result for an error */
-    unsigned long long count_in = 0, count_out;
+    long long count_in = 0, count_out, bytesToOffset = -1;
 
     assert(f_in != NULL); assert(f_out != NULL);
     assert(ctx != NULL);
@@ -81,22 +82,48 @@ compress_file_internal(FILE* f_in, FILE* f_out,
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
+      size_t compressedSize;
+      long long inSize = IN_CHUNK_SIZE;
+      if (uncOffset >= 0) {
+        bytesToOffset = uncOffset - count_in;
+
+        /* read only remaining bytes to offset position */
+        if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) {
+          inSize = bytesToOffset;
+        }
+      }
+
+      /* input data is at uncompressed data offset */
+      if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) {
+        size_t const readSize = fread(inBuff, 1, inSize, f_unc);
+        if (readSize == 0) {
+          uncOffset = -1;
+          continue;
+        }
+        count_in += readSize;
+        compressedSize = LZ4F_uncompressedUpdate(ctx,
+                                             outBuff, outCapacity,
+                                             inBuff, readSize,
+                                             NULL);
+      } else {
+        size_t const readSize = fread(inBuff, 1, inSize, f_in);
         if (readSize == 0) break; /* nothing left to read from input file */
         count_in += readSize;
-
-        size_t const compressedSize = LZ4F_compressUpdate(ctx,
+        compressedSize = LZ4F_compressUpdate(ctx,
                                                 outBuff, outCapacity,
                                                 inBuff, readSize,
                                                 NULL);
-        if (LZ4F_isError(compressedSize)) {
-            printf("Compression failed: error %u \n", (unsigned)compressedSize);
-            return result;
-        }
 
-        printf("Writing %u bytes\n", (unsigned)compressedSize);
-        safe_fwrite(outBuff, 1, compressedSize, f_out);
-        count_out += compressedSize;
+      }
+
+      if (LZ4F_isError(compressedSize)) {
+        printf("Compression failed: error %u \n", (unsigned)compressedSize);
+        return result;
+      }
+
+      printf("Writing %u bytes\n", (unsigned)compressedSize);
+      safe_fwrite(outBuff, 1, compressedSize, f_out);
+      count_out += compressedSize;
     }
 
     /* flush whatever remains within internal buffers */
@@ -120,7 +147,8 @@ compress_file_internal(FILE* f_in, FILE* f_out,
 }
 
 static compressResult_t
-compress_file(FILE* f_in, FILE* f_out)
+compress_file(FILE* f_in, FILE* f_out,
+              FILE* f_unc, int uncOffset)
 {
     assert(f_in != NULL);
     assert(f_out != NULL);
@@ -137,7 +165,8 @@ compress_file(FILE* f_in, FILE* f_out)
         result = compress_file_internal(f_in, f_out,
                                         ctx,
                                         src, IN_CHUNK_SIZE,
-                                        outbuff, outbufCapacity);
+                                        outbuff, outbufCapacity,
+                                        f_unc, uncOffset);
     } else {
         printf("error : resource allocation failed \n");
     }
@@ -305,52 +334,106 @@ static int decompress_file(FILE* f_in, FILE* f_out)
 }
 
 
-int compareFiles(FILE* fp0, FILE* fp1)
+int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset)
 {
     int result = 0;
+    long bytesRead = 0;
+    long bytesToOffset = -1;
+    long b1Size = 1024;
 
     while (result==0) {
+        char b1[b1Size];
+        size_t r1;
+        size_t bytesToRead = sizeof b1;
+        if (uncOffset >= 0) {
+          bytesToOffset = uncOffset - bytesRead;
+
+          /* read remainder to offset */
+          if (bytesToOffset < b1Size) {
+            bytesToRead = bytesToOffset;
+          }
+        }
+
         char b0[1024];
-        char b1[1024];
-        size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
-        size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
+        size_t r0;
+        if (bytesToOffset <= 0 && fpUnc) {
+          bytesToRead = sizeof b1;
+          r0 = fread(b0, 1,bytesToRead, fpUnc);
+        } else {
+          r0 = fread(b0, 1, bytesToRead, fp0);
+        }
+
+        r1 = fread(b1, 1, r0, fp1);
 
         result = (r0 != r1);
         if (!r0 || !r1) break;
         if (!result) result = memcmp(b0, b1, r0);
+
+        bytesRead += r1;
     }
 
     return result;
 }
 
 
-int main(int argc, const char **argv) {
+int main(int argc, char **argv) {
     char inpFilename[256] = { 0 };
     char lz4Filename[256] = { 0 };
     char decFilename[256] = { 0 };
 
+    int uncOffset = -1;
+    char uncFilename[256] = { 0 };
+    int opt;
+
     if (argc < 2) {
         printf("Please specify input filename\n");
-        return 0;
+        return EXIT_FAILURE;
     }
 
     snprintf(inpFilename, 256, "%s", argv[1]);
     snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
     snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
 
+    while ((opt = getopt(argc, argv, "o:d:")) != -1) {
+      switch (opt) {
+      case 'd':
+        snprintf(uncFilename, 256, "%s", optarg);
+        break;
+      case 'o':
+        uncOffset = atoi(optarg);
+        break;
+      default:
+        printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]);
+        printf("-o uncompressed data offset\n");
+        printf("   inject uncompressed data at this offset into the lz4 file\n");
+        printf("-d uncompressed file\n");
+        printf("   file to inject without compression into the lz4 file\n");
+        return EXIT_FAILURE;
+      }
+    }
+
     printf("inp = [%s]\n", inpFilename);
     printf("lz4 = [%s]\n", lz4Filename);
     printf("dec = [%s]\n", decFilename);
+    if (uncOffset > 0) {
+      printf("unc = [%s]\n", uncFilename);
+      printf("ofs = [%i]\n", uncOffset);
+    }
 
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
+        FILE* const uncFp = fopen(uncFilename, "rb");
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
-        compressResult_t const ret = compress_file(inpFp, outFp);
+        compressResult_t const ret = compress_file(
+            inpFp, outFp,
+            uncFp, uncOffset);
 
         fclose(outFp);
         fclose(inpFp);
+        if (uncFp)
+          fclose(uncFp);
 
         if (ret.error) {
             printf("compress : failed with code %i\n", ret.error);
@@ -383,12 +466,16 @@ int main(int argc, const char **argv) {
     /* verify */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const decFp = fopen(decFilename, "rb");
+        FILE* const uncFp = fopen(uncFilename, "rb");
 
         printf("verify : %s <-> %s\n", inpFilename, decFilename);
-        int const cmp = compareFiles(inpFp, decFp);
+        int const cmp = compareFiles(inpFp, decFp,
+                                     uncFp, uncOffset);
 
         fclose(decFp);
         fclose(inpFp);
+        if (uncFp)
+          fclose(uncFp);
 
         if (cmp) {
             printf("corruption detected : decompressed file differs from original\n");
diff --git a/lib/lz4.c b/lib/lz4.c
index a2272cf..16ed3d3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1679,6 +1679,15 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     return result;
 }
 
+int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize)
+{
+  LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
+
+  if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
+  if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
+
+  return dictSize;
+}
 
 /*! LZ4_saveDict() :
  *  If previously compressed data block is not guaranteed to remain available at its memory location,
@@ -1690,12 +1699,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
 {
     LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
-
+    dictSize = LZ4_DictSize(LZ4_dict, dictSize);
     DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);
 
-    if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
-    if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
-
     if (safeBuffer == NULL) assert(dictSize == 0);
     if (dictSize > 0) {
         const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
diff --git a/lib/lz4.h b/lib/lz4.h
index 6c068c6..1e793fd 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -346,6 +346,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  */
 LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
+LZ4LIB_API int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize);
+
 /*! LZ4_saveDict() :
  *  If last 64KB data cannot be guaranteed to remain available at its current memory location,
  *  save it into a safer place (char* safeBuffer).
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a0275ca..bcf9629 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -129,7 +129,8 @@ static int g_debuglog_enable = 1;
 **************************************/
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include <stdint.h>
-  typedef  uint8_t BYTE;
+#include <stdio.h>
+typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef uint32_t U32;
   typedef  int32_t S32;
@@ -236,6 +237,7 @@ typedef struct LZ4F_cctx_s
     void*  lz4CtxPtr;
     U16    lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
     U16    lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+    LZ4F_blockCompression_t  blockCompression;
 } LZ4F_cctx_t;
 
 
@@ -757,14 +759,27 @@ static size_t LZ4F_makeBlock(void* dst,
                        const void* src, size_t srcSize,
                              compressFunc_t compress, void* lz4ctx, int level,
                        const LZ4F_CDict* cdict,
-                             LZ4F_blockChecksum_t crcFlag)
+                             LZ4F_blockChecksum_t crcFlag,
+                             LZ4F_blockCompression_t blockCompression)
 {
     BYTE* const cSizePtr = (BYTE*)dst;
-    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
-                                      (int)(srcSize), (int)(srcSize-1),
-                                      level, cdict);
-    if (cSize == 0) {  /* compression failed */
-        DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+    U32 cSize;
+    if (compress != NULL) {
+      cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
+                            (int)(srcSize), (int)(srcSize-1),
+                            level, cdict);
+    } else {
+      cSize = (U32)srcSize;
+      /* force no compression if compress callback is null */
+      blockCompression = LZ4B_UNCOMPRESSED;
+    }
+
+    if (cSize == 0) { /* compression failed */
+      DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+      blockCompression = LZ4B_UNCOMPRESSED;
+    }
+
+    if (blockCompression == LZ4B_UNCOMPRESSED) {
         cSize = (U32)srcSize;
         LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
         memcpy(cSizePtr+BHSize, src, srcSize);
@@ -824,33 +839,48 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
     return LZ4F_compressBlockHC_continue;
 }
 
+static int LZ4F_maxDictSize(void) {
+  return 64 KB;
+}
+
 /* Save history (up to 64KB) into @tmpBuff */
 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
 {
     if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-        return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
-    return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
+        return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
+    return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
+}
+
+static int LZ4F_localDictSize(LZ4F_cctx_t* cctxPtr)
+{
+  if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+    return LZ4_DictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
+  return LZ4_DictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
 }
 
 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
 
 static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
 
-/*! LZ4F_compressUpdate() :
+
+ /*! LZ4F_compressUpdateImpl() :
  *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
  *  When successful, the function always entirely consumes @srcBuffer.
  *  src data is either buffered or compressed into @dstBuffer.
- * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ *  If the block compression does not match the compression of the previous block, the old data is flushed
+ *  and operations continue with the new compression mode.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
  * @compressOptionsPtr is optional : provide NULL to mean "default".
  * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  After an error, the state is left in a UB state, and must be re-initialized.
  */
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
-                           void* dstBuffer, size_t dstCapacity,
+static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
+                     void* dstBuffer, size_t dstCapacity,
                      const void* srcBuffer, size_t srcSize,
-                     const LZ4F_compressOptions_t* compressOptionsPtr)
-{
+                     const LZ4F_compressOptions_t* compressOptionsPtr,
+                     LZ4F_blockCompression_t blockCompression)
+  {
     size_t const blockSize = cctxPtr->maxBlockSize;
     const BYTE* srcPtr = (const BYTE*)srcBuffer;
     const BYTE* const srcEnd = srcPtr + srcSize;
@@ -858,49 +888,62 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
-
+    size_t bytesWritten = 0;
     DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
 
+    /* flush currently written block, to continue with new block compression */
+    if (cctxPtr->blockCompression != blockCompression) {
+      bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+      cctxPtr->blockCompression = blockCompression;
+    }
+
     RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);   /* state must be initialized and waiting for next block */
-    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
-        RETURN_ERROR(dstMaxSize_tooSmall);
+
+    if (blockCompression == LZ4B_COMPRESSED &&
+        dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
+      RETURN_ERROR(dstMaxSize_tooSmall);
+
+    if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
+      RETURN_ERROR(dstMaxSize_tooSmall);
+
     if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
 
     /* complete tmp buffer */
     if (cctxPtr->tmpInSize > 0) {   /* some data already within tmp buffer */
-        size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
-        assert(blockSize > cctxPtr->tmpInSize);
-        if (sizeToCopy > srcSize) {
-            /* add src to tmpIn buffer */
-            memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
-            srcPtr = srcEnd;
-            cctxPtr->tmpInSize += srcSize;
-            /* still needs some CRC */
-        } else {
-            /* complete tmpIn block and then compress it */
-            lastBlockCompressed = fromTmpBuffer;
-            memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
-            srcPtr += sizeToCopy;
-
-            dstPtr += LZ4F_makeBlock(dstPtr,
-                                     cctxPtr->tmpIn, blockSize,
-                                     compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
-                                     cctxPtr->cdict,
-                                     cctxPtr->prefs.frameInfo.blockChecksumFlag);
-
-            if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
-            cctxPtr->tmpInSize = 0;
-    }   }
+      size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
+      assert(blockSize > cctxPtr->tmpInSize);
+      if (sizeToCopy > srcSize) {
+        /* add src to tmpIn buffer */
+        memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
+        srcPtr = srcEnd;
+        cctxPtr->tmpInSize += srcSize;
+        /* still needs some CRC */
+      } else {
+        /* complete tmpIn block and then compress it */
+        lastBlockCompressed = fromTmpBuffer;
+        memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
+        srcPtr += sizeToCopy;
 
-    while ((size_t)(srcEnd - srcPtr) >= blockSize) {
-        /* compress full blocks */
-        lastBlockCompressed = fromSrcBuffer;
         dstPtr += LZ4F_makeBlock(dstPtr,
-                                 srcPtr, blockSize,
+                                 cctxPtr->tmpIn, blockSize,
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                  cctxPtr->cdict,
-                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
-        srcPtr += blockSize;
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
+
+        if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
+        cctxPtr->tmpInSize = 0;
+      }
+    }
+
+    while ((size_t)(srcEnd - srcPtr) >= blockSize) {
+      /* compress full blocks */
+      lastBlockCompressed = fromSrcBuffer;
+      dstPtr += LZ4F_makeBlock(dstPtr,
+                               srcPtr, blockSize,
+                               compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+                               cctxPtr->cdict,
+                               cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
+      srcPtr += blockSize;
     }
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
@@ -910,19 +953,29 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
                                  srcPtr, (size_t)(srcEnd - srcPtr),
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                  cctxPtr->cdict,
-                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
-        srcPtr  = srcEnd;
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag,
+                                 blockCompression);
+      srcPtr = srcEnd;
     }
 
     /* preserve dictionary within @tmpBuff whenever necessary */
     if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
-        if (compressOptionsPtr->stableSrc) {
-            cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
+      if (compressOptionsPtr->stableSrc) {
+        cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
+      } else {
+        int realDictSize;
+        if (blockCompression == LZ4B_COMPRESSED) {
+          realDictSize = LZ4F_localSaveDict(cctxPtr);
         } else {
-            int const realDictSize = LZ4F_localSaveDict(cctxPtr);
-            assert(0 <= realDictSize && realDictSize <= 64 KB);
-            cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+          /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate
+           * this is only relevant if linked block mode
+           * */
+          realDictSize = LZ4F_localDictSize(cctxPtr);
         }
+
+        assert(0 <= realDictSize && realDictSize <= 64 KB);
+        cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+      }
     }
 
     /* keep tmpIn within limits */
@@ -931,24 +984,75 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     {
         /* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
          * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
-        int const realDictSize = LZ4F_localSaveDict(cctxPtr);
+        int realDictSize;
+        if (blockCompression == LZ4B_COMPRESSED) {
+          realDictSize = LZ4F_localSaveDict(cctxPtr);
+        } else {
+          /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate*/
+          realDictSize = LZ4F_maxDictSize();
+        }
         cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
         assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
     }
 
     /* some input data left, necessarily < blockSize */
     if (srcPtr < srcEnd) {
-        /* fill tmp buffer */
-        size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
-        memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
-        cctxPtr->tmpInSize = sizeToCopy;
+       /* fill tmp buffer */
+       size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
+       memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
+       cctxPtr->tmpInSize = sizeToCopy;
     }
 
     if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
-        (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
+      (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
 
     cctxPtr->totalInSize += srcSize;
-    return (size_t)(dstPtr - dstStart);
+    return bytesWritten + (size_t)(dstPtr - dstStart);
+}
+
+/*! LZ4F_compressUpdate() :
+ *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ *  When successful, the function always entirely consumes @srcBuffer.
+ *  src data is either buffered or compressed into @dstBuffer.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ *  After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+                           void* dstBuffer, size_t dstCapacity,
+                     const void* srcBuffer, size_t srcSize,
+                     const LZ4F_compressOptions_t* compressOptionsPtr)
+{
+   return LZ4F_compressUpdateImpl(cctxPtr,
+                                 dstBuffer, dstCapacity,
+                                 srcBuffer, srcSize,
+                                 compressOptionsPtr, LZ4B_COMPRESSED);
+}
+
+/*! LZ4F_compressUpdate() :
+ *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ *  When successful, the function always entirely consumes @srcBuffer.
+ *  src data is either buffered or compressed into @dstBuffer.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ *  After an error, the state is left in a UB state, and must be re-initialized.
+ */
+LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+                                               void* dstBuffer, size_t dstCapacity,
+                                               const void* srcBuffer, size_t srcSize,
+                                               const LZ4F_compressOptions_t* compressOptionsPtr) {
+  return LZ4F_compressUpdateImpl(cctxPtr,
+                                 dstBuffer, dstCapacity,
+                                 srcBuffer, srcSize,
+                                 compressOptionsPtr, LZ4B_UNCOMPRESSED);
 }
 
 
@@ -981,7 +1085,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
                              cctxPtr->tmpIn, cctxPtr->tmpInSize,
                              compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                              cctxPtr->cdict,
-                             cctxPtr->prefs.frameInfo.blockChecksumFlag);
+                             cctxPtr->prefs.frameInfo.blockChecksumFlag, cctxPtr->blockCompression);
     assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));
 
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 74f19cd..18d33e1 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -160,6 +160,11 @@ typedef enum {
     LZ4F_OBSOLETE_ENUM(skippableFrame)
 } LZ4F_frameType_t;
 
+typedef enum {
+  LZ4B_COMPRESSED,
+  LZ4B_UNCOMPRESSED
+} LZ4F_blockCompression_t;
+
 #ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
 typedef LZ4F_blockSizeID_t blockSizeID_t;
 typedef LZ4F_blockMode_t blockMode_t;
@@ -303,6 +308,8 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
  *  This value is provided by LZ4F_compressBound().
  *  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
  *  After an error, the state is left in a UB state, and must be re-initialized or freed.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
  * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
@@ -312,6 +319,22 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
                                  const void* srcBuffer, size_t srcSize,
                                  const LZ4F_compressOptions_t* cOptPtr);
 
+/*! LZ4F_uncompressedUpdate() :
+ *  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ *  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ *  no compression is done for this operation
+ *  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ *  After an error, the state is left in a UB state, and must be re-initialized or freed.
+ *  If previously a compressed block was written, buffered data is flushed
+ *  before appending uncompressed data is continued.
+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                       const void* srcBuffer, size_t srcSize,
+                                       const LZ4F_compressOptions_t* cOptPtr);
 /*! LZ4F_flush() :
  *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
  *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 99650a6..da806ef 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1154,6 +1154,16 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
     return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);
 }
 
+int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
+  LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
+  int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+  DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
+  assert(prefixSize >= 0);
+  if (dictSize > 64 KB) dictSize = 64 KB;
+  if (dictSize < 4) dictSize = 0;
+  if (dictSize > prefixSize) dictSize = prefixSize;
+  return dictSize;
+}
 
 
 /* LZ4_saveDictHC :
@@ -1164,12 +1174,7 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
 int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
 {
     LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
-    int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
-    DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
-    assert(prefixSize >= 0);
-    if (dictSize > 64 KB) dictSize = 64 KB;
-    if (dictSize < 4) dictSize = 0;
-    if (dictSize > prefixSize) dictSize = prefixSize;
+    dictSize = LZ4_DictHCSize(LZ4_streamHCPtr, dictSize);
     if (safeBuffer == NULL) assert(dictSize == 0);
     if (dictSize > 0)
         memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index f4afc9b..11671dc 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -173,6 +173,8 @@ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr
                                            const char* src, char* dst,
                                                  int* srcSizePtr, int targetDstSize);
 
+LZ4LIB_API int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
+
 LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
 
 
-- 
cgit v0.12


From 62f6cef564a9478b92e6dbd0faa81eaa1a90b34f Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Fri, 10 Jun 2022 06:00:38 +0000
Subject: review: Fix review findings

This commit fixes the review findings

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 doc/lz4_manual.html      | 12 ++++++++++++
 doc/lz4frame_manual.html | 37 +++++++++++++++++++++++++------------
 lib/lz4.c                | 14 ++++++++++++--
 lib/lz4.h                | 13 +++++++++++--
 lib/lz4frame.c           |  7 +++++--
 lib/lz4frame.h           |  5 -----
 lib/lz4hc.c              | 14 ++++++++++++--
 lib/lz4hc.h              | 14 ++++++++++++--
 8 files changed, 89 insertions(+), 27 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 037cfc0..13c1ae6 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -391,6 +391,18 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  
 </p></pre><BR>
 
+<pre><b>LZ4LIB_STATIC_API int LZ4_getDictSize (LZ4_stream_t* LZ4_dict, int dictSize);
+</b><p> Get the size of the dictionary. This can be used for adding data without
+ compression to the LZ4 archive. If linked blocked mode is used the memory
+ of the dictionary is kept free.
+ This way uncompressed data does not influence the effectiveness of the
+ dictionary.
+ @param LZ4_dict Pointer to the dictionary to get the size of.
+ @param dictSize The maximum dictionary size. (Normally 64 KB).
+ @return The size of the dictionary.
+ 
+</p></pre><BR>
+
 <pre><b></b><p>
  It's possible to have input and output sharing the same buffer,
  for highly constrained memory environments.
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 9490b51..b47a92f 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -75,11 +75,6 @@
     LZ4F_OBSOLETE_ENUM(skippableFrame)
 } LZ4F_frameType_t;
 </b></pre><BR>
-<pre><b>typedef enum {
-  LZ4B_COMPRESSED,
-  LZ4B_UNCOMPRESSED,
-} LZ4F_blockCompression_t;
-</b></pre><BR>
 <pre><b>typedef struct {
   LZ4F_blockSizeID_t     blockSizeID;         </b>/* max64KB, max256KB, max1MB, max4MB; 0 == default */<b>
   LZ4F_blockMode_t       blockMode;           </b>/* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */<b>
@@ -189,22 +184,40 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
   This value is provided by LZ4F_compressBound().
   If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
   After an error, the state is left in a UB state, and must be re-initialized or freed.
+  If previously an uncompressed block was written, buffered data is flushed
+  before appending compressed data is continued.
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx,
+<pre><b>size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                       const void* srcBuffer, size_t srcSize,
+                                       const LZ4F_compressOptions_t* cOptPtr);
+/*! LZ4F_flush() :
+ *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
+ *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
+ * `dstCapacity` must be large enough to ensure the operation will be successful.
+ * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
+ * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ *  Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).
+ */
+size_t LZ4F_flush(LZ4F_cctx* cctx,
                               void* dstBuffer, size_t dstCapacity,
                         const LZ4F_compressOptions_t* cOptPtr);
-</b><p>  When data must be generated and sent immediately, without waiting for a block to be completely filled,
-  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
- `dstCapacity` must be large enough to ensure the operation will be successful.
- `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
+</b><p>  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+  no compression is done for this operation
+  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+  After an error, the state is left in a UB state, and must be re-initialized or freed.
+  If previously a compressed block was written, buffered data is flushed
+  before appending uncompressed data is continued.
+ `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
-  Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).
  
 </p></pre><BR>
 
diff --git a/lib/lz4.c b/lib/lz4.c
index 16ed3d3..932e2cd 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1679,7 +1679,17 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     return result;
 }
 
-int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize)
+/*! LZ4_getDictSize():
+ * Get the size of the dictionary. This can be used for adding data without
+ * compression to the LZ4 archive. If linked blocked mode is used the memory
+ * of the dictionary is kept free.
+ * This way uncompressed data does not influence the effectiveness of the
+ * dictionary.
+ * @param LZ4_dict Pointer to the dictionary to get the size of.
+ * @param dictSize The maximum dictionary size. (Normally 64 KB).
+ * @return The size of the dictionary.
+ */
+int LZ4_getDictSize (LZ4_stream_t* LZ4_dict, int dictSize)
 {
   LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
 
@@ -1699,7 +1709,7 @@ int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize)
 int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
 {
     LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
-    dictSize = LZ4_DictSize(LZ4_dict, dictSize);
+    dictSize = LZ4_getDictSize(LZ4_dict, dictSize);
     DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);
 
     if (safeBuffer == NULL) assert(dictSize == 0);
diff --git a/lib/lz4.h b/lib/lz4.h
index 1e793fd..f2a529f 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -346,8 +346,6 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  */
 LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
-LZ4LIB_API int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize);
-
 /*! LZ4_saveDict() :
  *  If last 64KB data cannot be guaranteed to remain available at its current memory location,
  *  save it into a safer place (char* safeBuffer).
@@ -511,6 +509,17 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c
  */
 LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
 
+/*! LZ4_getDictSize():
+ * Get the size of the dictionary. This can be used for adding data without
+ * compression to the LZ4 archive. If linked blocked mode is used the memory
+ * of the dictionary is kept free.
+ * This way uncompressed data does not influence the effectiveness of the
+ * dictionary.
+ * @param LZ4_dict Pointer to the dictionary to get the size of.
+ * @param dictSize The maximum dictionary size. (Normally 64 KB).
+ * @return The size of the dictionary.
+ */
+LZ4LIB_STATIC_API int LZ4_getDictSize (LZ4_stream_t* LZ4_dict, int dictSize);
 
 /*! In-place compression and decompression
  *
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index bcf9629..0c78a1f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -221,6 +221,9 @@ static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE;  /* block footer : checks
 /*-************************************
 *  Structures and local types
 **************************************/
+
+typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t;
+
 typedef struct LZ4F_cctx_s
 {
     LZ4F_preferences_t prefs;
@@ -854,8 +857,8 @@ static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
 static int LZ4F_localDictSize(LZ4F_cctx_t* cctxPtr)
 {
   if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-    return LZ4_DictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
-  return LZ4_DictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
+    return LZ4_getDictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
+  return LZ4_getDictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
 }
 
 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 18d33e1..20bfb8b 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -160,11 +160,6 @@ typedef enum {
     LZ4F_OBSOLETE_ENUM(skippableFrame)
 } LZ4F_frameType_t;
 
-typedef enum {
-  LZ4B_COMPRESSED,
-  LZ4B_UNCOMPRESSED
-} LZ4F_blockCompression_t;
-
 #ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
 typedef LZ4F_blockSizeID_t blockSizeID_t;
 typedef LZ4F_blockMode_t blockMode_t;
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index da806ef..bf6294d 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1154,7 +1154,17 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
     return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);
 }
 
-int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
+/*! LZ4_getDictHCSize():
+ * Get the size of the dictionary. This can be used for adding data without
+ * compression to the LZ4 archive. If linked blocked mode is used the memory
+ * of the dictionary is kept free.
+ * This way uncompressed data does not influence the effectiveness of the
+ * dictionary.
+ * @param LZ4_dict Pointer to the dictionary to get the size of.
+ * @param dictSize The maximum dictionary size. (Normally 64 KB).
+ * @return The size of the dictionary.
+ */
+int LZ4_getDictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
   LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
   int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
   DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
@@ -1174,7 +1184,7 @@ int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
 int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
 {
     LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
-    dictSize = LZ4_DictHCSize(LZ4_streamHCPtr, dictSize);
+    dictSize = LZ4_getDictHCSize(LZ4_streamHCPtr, dictSize);
     if (safeBuffer == NULL) assert(dictSize == 0);
     if (dictSize > 0)
         memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 11671dc..e62dfa7 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -173,8 +173,6 @@ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr
                                            const char* src, char* dst,
                                                  int* srcSizePtr, int targetDstSize);
 
-LZ4LIB_API int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
-
 LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
 
 
@@ -407,6 +405,18 @@ LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary(
           LZ4_streamHC_t *working_stream,
     const LZ4_streamHC_t *dictionary_stream);
 
+/*! LZ4_getDictHCSize():
+ * Get the size of the dictionary. This can be used for adding data without
+ * compression to the LZ4 archive. If linked blocked mode is used the memory
+ * of the dictionary is kept free.
+ * This way uncompressed data does not influence the effectiveness of the
+ * dictionary.
+ * @param LZ4_dict Pointer to the dictionary to get the size of.
+ * @param dictSize The maximum dictionary size. (Normally 64 KB).
+ * @return The size of the dictionary.
+ */
+LZ4LIB_STATIC_API int LZ4_getDictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
+
 #if defined (__cplusplus)
 }
 #endif
-- 
cgit v0.12


From 5c7382798eb9b5af9eaa6c66f018c8efcb62f8d5 Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Fri, 10 Jun 2022 06:37:57 +0000
Subject: review: Fix review findings

This commit fixes the review findings

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 doc/lz4frame_manual.html | 47 ++++++++++++++++++++++++-----------------------
 examples/frameCompress.c |  1 +
 lib/lz4frame.h           | 33 +++++++++++++++++----------------
 ossfuzz/fuzz.h           |  5 +++++
 4 files changed, 47 insertions(+), 39 deletions(-)

diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index b47a92f..c55c6e9 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -192,32 +192,16 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
  
 </p></pre><BR>
 
-<pre><b>size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
-                                       void* dstBuffer, size_t dstCapacity,
-                                       const void* srcBuffer, size_t srcSize,
-                                       const LZ4F_compressOptions_t* cOptPtr);
-/*! LZ4F_flush() :
- *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
- *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
- * `dstCapacity` must be large enough to ensure the operation will be successful.
- * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
- * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
- *           or an error code if it fails (which can be tested using LZ4F_isError())
- *  Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).
- */
-size_t LZ4F_flush(LZ4F_cctx* cctx,
+<pre><b>size_t LZ4F_flush(LZ4F_cctx* cctx,
                               void* dstBuffer, size_t dstCapacity,
                         const LZ4F_compressOptions_t* cOptPtr);
-</b><p>  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
-  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
-  no compression is done for this operation
-  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
-  After an error, the state is left in a UB state, and must be re-initialized or freed.
-  If previously a compressed block was written, buffered data is flushed
-  before appending uncompressed data is continued.
- `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
- @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+</b><p>  When data must be generated and sent immediately, without waiting for a block to be completely filled,
+  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
+ `dstCapacity` must be large enough to ensure the operation will be successful.
+ `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default.
+ @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx)
            or an error code if it fails (which can be tested using LZ4F_isError())
+  Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr).
  
 </p></pre><BR>
 
@@ -360,6 +344,23 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 <pre><b>typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM)
               _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes;
 </b></pre><BR>
+<pre><b>LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+                                                  void* dstBuffer, size_t dstCapacity,
+                                                  const void* srcBuffer, size_t srcSize,
+                                                  const LZ4F_compressOptions_t* cOptPtr);
+</b><p>  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+  no compression is done for this operation
+  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+  After an error, the state is left in a UB state, and must be re-initialized or freed.
+  If previously a compressed block was written, buffered data is flushed
+  before appending uncompressed data is continued.
+ `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+           or an error code if it fails (which can be tested using LZ4F_isError())
+ 
+</p></pre><BR>
+
 <a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre>
 
 <pre><b>LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize);
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 3219014..25ff729 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -13,6 +13,7 @@
 
 #include <getopt.h>
 #include <lz4frame.h>
+#include <lz4frame_static.h>
 
 #define IN_CHUNK_SIZE  (16*1024)
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 20bfb8b..7d81fa0 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -314,22 +314,6 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
                                  const void* srcBuffer, size_t srcSize,
                                  const LZ4F_compressOptions_t* cOptPtr);
 
-/*! LZ4F_uncompressedUpdate() :
- *  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
- *  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
- *  no compression is done for this operation
- *  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
- *  After an error, the state is left in a UB state, and must be re-initialized or freed.
- *  If previously a compressed block was written, buffered data is flushed
- *  before appending uncompressed data is continued.
- * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
- * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
- *           or an error code if it fails (which can be tested using LZ4F_isError())
- */
-LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
-                                       void* dstBuffer, size_t dstCapacity,
-                                       const void* srcBuffer, size_t srcSize,
-                                       const LZ4F_compressOptions_t* cOptPtr);
 /*! LZ4F_flush() :
  *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
  *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
@@ -560,6 +544,23 @@ LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
 
 LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);
 
+/*! LZ4F_uncompressedUpdate() :
+ *  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ *  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ *  no compression is done for this operation
+ *  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ *  After an error, the state is left in a UB state, and must be re-initialized or freed.
+ *  If previously a compressed block was written, buffered data is flushed
+ *  before appending uncompressed data is continued.
+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+                                                  void* dstBuffer, size_t dstCapacity,
+                                                  const void* srcBuffer, size_t srcSize,
+                                                  const LZ4F_compressOptions_t* cOptPtr);
+
 /**********************************
  *  Bulk processing dictionary API
  *********************************/
diff --git a/ossfuzz/fuzz.h b/ossfuzz/fuzz.h
index eefac63..917a304 100644
--- a/ossfuzz/fuzz.h
+++ b/ossfuzz/fuzz.h
@@ -41,6 +41,11 @@ extern "C" {
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size);
 
+/**
+ * Test if injection of uncompressed data into a stream is working properly
+ */
+int LLVMFuzzerUncompressedDataInjection(const uint8_t *data, size_t size)
+
 #ifdef __cplusplus
 }
 #endif
-- 
cgit v0.12


From 1738b50443d1446c9c80ebd7c9272427f2644809 Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Fri, 10 Jun 2022 19:45:21 +0000
Subject: fuzz-test: add fuzz test for uncompressed api

add a fuzzing test for uncompressed frame api

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 lib/lz4frame.c                                 |   3 +-
 ossfuzz/Makefile                               |   1 +
 ossfuzz/fuzz.h                                 |   5 -
 ossfuzz/round_trip_frame_uncompressed_fuzzer.c | 127 +++++++++++++++++++++++++
 4 files changed, 129 insertions(+), 7 deletions(-)
 create mode 100644 ossfuzz/round_trip_frame_uncompressed_fuzzer.c

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 0c78a1f..c7e0595 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -129,8 +129,7 @@ static int g_debuglog_enable = 1;
 **************************************/
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include <stdint.h>
-#include <stdio.h>
-typedef  uint8_t BYTE;
+  typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef uint32_t U32;
   typedef  int32_t S32;
diff --git a/ossfuzz/Makefile b/ossfuzz/Makefile
index 2ec1675..deb2938 100644
--- a/ossfuzz/Makefile
+++ b/ossfuzz/Makefile
@@ -45,6 +45,7 @@ FUZZERS := \
 	round_trip_hc_fuzzer \
 	compress_frame_fuzzer \
 	round_trip_frame_fuzzer \
+	round_trip_frame_uncompressed_fuzzer \
 	decompress_frame_fuzzer
 
 .PHONY: all
diff --git a/ossfuzz/fuzz.h b/ossfuzz/fuzz.h
index 917a304..eefac63 100644
--- a/ossfuzz/fuzz.h
+++ b/ossfuzz/fuzz.h
@@ -41,11 +41,6 @@ extern "C" {
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size);
 
-/**
- * Test if injection of uncompressed data into a stream is working properly
- */
-int LLVMFuzzerUncompressedDataInjection(const uint8_t *data, size_t size)
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/ossfuzz/round_trip_frame_uncompressed_fuzzer.c b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
new file mode 100644
index 0000000..631d149
--- /dev/null
+++ b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
@@ -0,0 +1,127 @@
+/**
+ * This fuzz target performs a lz4 round-trip test (compress & decompress),
+ * compares the result with the original, and calls abort() on corruption.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fuzz_data_producer.h"
+#include "fuzz_helpers.h"
+#include "lz4.h"
+#include "lz4_helpers.h"
+#include "lz4frame.h"
+#include "lz4frame_static.h"
+
+static void decompress_data(LZ4F_dctx *dctx, void *src, void *dst,
+                           size_t dstCapacity, size_t readSize) {
+  size_t ret = 1;
+  const void *srcPtr = (const char *)src;
+  void *dstPtr = (char *)dst;
+  const void *const srcEnd = (const char *)srcPtr + readSize;
+
+  while (ret != 0) {
+    while (srcPtr < srcEnd && ret != 0) {
+      /* Any data within dst has been flushed at this stage */
+      size_t dstSize = dstCapacity;
+      size_t srcSize = (const char *)srcEnd - (const char *)srcPtr;
+      ret = LZ4F_decompress(dctx, dstPtr, &dstSize, srcPtr, &srcSize,
+                            /* LZ4F_decompressOptions_t */ NULL);
+      FUZZ_ASSERT(!LZ4F_isError(ret));
+
+      /* Update input */
+      srcPtr = (const char *)srcPtr + srcSize;
+      dstPtr = (char *)dstPtr + dstSize;
+    }
+
+    FUZZ_ASSERT(srcPtr <= srcEnd);
+  }
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+  LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer);
+  size = FUZZ_dataProducer_remainingBytes(producer);
+
+  uint8_t *uncompressedData = malloc(size);
+  size_t uncompressedOffset = rand() % (size + 1);
+
+  FUZZ_dataProducer_t *uncompressedProducer =
+      FUZZ_dataProducer_create(uncompressedData, size);
+  size_t uncompressedSize =
+      FUZZ_dataProducer_remainingBytes(uncompressedProducer);
+
+  size_t const dstCapacity =
+      LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs) +
+      uncompressedSize;
+  char *const dst = (char *)malloc(dstCapacity);
+  size_t rtCapacity = dstCapacity;
+  char *const rt = (char *)malloc(rtCapacity);
+
+  FUZZ_ASSERT(dst);
+  FUZZ_ASSERT(rt);
+
+  /* Compression must succeed and round trip correctly. */
+  LZ4F_compressionContext_t ctx;
+  size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+  FUZZ_ASSERT(!LZ4F_isError(ctxCreation));
+
+  size_t const headerSize = LZ4F_compressBegin(ctx, dst, dstCapacity, &prefs);
+  FUZZ_ASSERT(!LZ4F_isError(headerSize));
+  size_t compressedSize = headerSize;
+
+  /* Compress data before uncompressed offset */
+  size_t lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity,
+                                         data, uncompressedOffset, NULL);
+  FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+  compressedSize += lz4Return;
+
+  /* Add uncompressed data */
+  lz4Return = LZ4F_uncompressedUpdate(ctx, dst + compressedSize, dstCapacity,
+                                      uncompressedData, uncompressedSize, NULL);
+  FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+  compressedSize += lz4Return;
+
+  /* Compress data after uncompressed offset */
+  lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity,
+                                  data + uncompressedOffset,
+                                  size - uncompressedOffset, NULL);
+  FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+  compressedSize += lz4Return;
+
+  /* Finish compression */
+  lz4Return = LZ4F_compressEnd(ctx, dst + compressedSize, dstCapacity, NULL);
+  FUZZ_ASSERT(!LZ4F_isError(lz4Return));
+  compressedSize += lz4Return;
+
+  LZ4F_decompressOptions_t opts;
+  memset(&opts, 0, sizeof(opts));
+  opts.stableDst = 1;
+  LZ4F_dctx *dctx;
+  LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+  FUZZ_ASSERT(dctx);
+
+  decompress_data(dctx, dst, rt, rtCapacity, compressedSize);
+
+  LZ4F_freeDecompressionContext(dctx);
+
+  char *const expectedData = (char *)malloc(size + uncompressedSize);
+  memcpy(expectedData, data, uncompressedOffset);
+  memcpy(expectedData + uncompressedOffset, uncompressedData, uncompressedSize);
+  memcpy(expectedData + uncompressedOffset + uncompressedSize,
+         data + uncompressedOffset, size - uncompressedOffset);
+
+  FUZZ_ASSERT_MSG(!memcmp(expectedData, rt, size), "Corruption!");
+  free(expectedData);
+
+  free(dst);
+  free(rt);
+  free(uncompressedData);
+
+  FUZZ_dataProducer_free(producer);
+  FUZZ_dataProducer_free(uncompressedProducer);
+  LZ4F_freeCompressionContext(ctx);
+  return 0;
+}
-- 
cgit v0.12


From 3c57d2f185c2b482fde69a4c5f04dbb48bacab58 Mon Sep 17 00:00:00 2001
From: Alexander Mohr <git@mohr.io>
Date: Sat, 11 Jun 2022 22:46:49 +0200
Subject: lz4frame: fix different linkage error

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 lib/lz4frame.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index c7e0595..788f19f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1047,7 +1047,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  After an error, the state is left in a UB state, and must be re-initialized.
  */
-LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
                                                void* dstBuffer, size_t dstCapacity,
                                                const void* srcBuffer, size_t srcSize,
                                                const LZ4F_compressOptions_t* compressOptionsPtr) {
-- 
cgit v0.12


From 9a42a9db94ff4c71de5590cae8d0f90339e6733b Mon Sep 17 00:00:00 2001
From: Alexander Mohr <git@mohr.io>
Date: Sat, 11 Jun 2022 23:14:56 +0200
Subject: dict-size: make lz4 context const

change the context to const to make
clear that the context is not modified
---
 doc/lz4_manual.html | 2 +-
 lib/lz4.c           | 4 ++--
 lib/lz4.h           | 2 +-
 lib/lz4hc.c         | 4 ++--
 lib/lz4hc.h         | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 13c1ae6..700cb84 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -391,7 +391,7 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  
 </p></pre><BR>
 
-<pre><b>LZ4LIB_STATIC_API int LZ4_getDictSize (LZ4_stream_t* LZ4_dict, int dictSize);
+<pre><b>LZ4LIB_STATIC_API int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize);
 </b><p> Get the size of the dictionary. This can be used for adding data without
  compression to the LZ4 archive. If linked blocked mode is used the memory
  of the dictionary is kept free.
diff --git a/lib/lz4.c b/lib/lz4.c
index 932e2cd..289162d 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1689,9 +1689,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
  * @param dictSize The maximum dictionary size. (Normally 64 KB).
  * @return The size of the dictionary.
  */
-int LZ4_getDictSize (LZ4_stream_t* LZ4_dict, int dictSize)
+int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize)
 {
-  LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
+  const LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
 
   if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
   if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
diff --git a/lib/lz4.h b/lib/lz4.h
index f2a529f..ce2288e 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -519,7 +519,7 @@ LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const
  * @param dictSize The maximum dictionary size. (Normally 64 KB).
  * @return The size of the dictionary.
  */
-LZ4LIB_STATIC_API int LZ4_getDictSize (LZ4_stream_t* LZ4_dict, int dictSize);
+LZ4LIB_STATIC_API int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize);
 
 /*! In-place compression and decompression
  *
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index bf6294d..b5bc880 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1164,8 +1164,8 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
  * @param dictSize The maximum dictionary size. (Normally 64 KB).
  * @return The size of the dictionary.
  */
-int LZ4_getDictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
-  LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
+int LZ4_getDictHCSize(const LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
+  const LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
   int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
   DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
   assert(prefixSize >= 0);
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index e62dfa7..3b31624 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -415,7 +415,7 @@ LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary(
  * @param dictSize The maximum dictionary size. (Normally 64 KB).
  * @return The size of the dictionary.
  */
-LZ4LIB_STATIC_API int LZ4_getDictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
+LZ4LIB_STATIC_API int LZ4_getDictHCSize(const LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
 
 #if defined (__cplusplus)
 }
-- 
cgit v0.12


From af447b22c80d25c8d88cc3147c120e027b18d0ae Mon Sep 17 00:00:00 2001
From: Alexander Mohr <git@mohr.io>
Date: Sun, 12 Jun 2022 00:41:55 +0200
Subject: meson: fix meson build

add static dependency to examples
---
 contrib/meson/meson/examples/meson.build | 2 +-
 lib/lz4frame.c                           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/meson/meson/examples/meson.build b/contrib/meson/meson/examples/meson.build
index 791bd94..f6b6c41 100644
--- a/contrib/meson/meson/examples/meson.build
+++ b/contrib/meson/meson/examples/meson.build
@@ -26,7 +26,7 @@ foreach e, src : examples
   executable(
     e,
     lz4_source_root / 'examples' / src,
-    dependencies: liblz4_dep,
+    dependencies: [liblz4_dep, liblz4_internal_dep],
     install: false
   )
 endforeach
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 788f19f..251521e 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1047,7 +1047,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  After an error, the state is left in a UB state, and must be re-initialized.
  */
-LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
                                                void* dstBuffer, size_t dstCapacity,
                                                const void* srcBuffer, size_t srcSize,
                                                const LZ4F_compressOptions_t* compressOptionsPtr) {
-- 
cgit v0.12


From 5065080664437efcb4215823b9ad95f5571a9d7c Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Mon, 13 Jun 2022 07:46:53 +0200
Subject: ossfuzz: extend fuzzing test to include linked blocks

fuzzing test now tests linked and independent blocks

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 ossfuzz/round_trip_frame_uncompressed_fuzzer.c | 29 +++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/ossfuzz/round_trip_frame_uncompressed_fuzzer.c b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
index 631d149..cf9cbb9 100644
--- a/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
+++ b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
@@ -15,7 +15,7 @@
 #include "lz4frame.h"
 #include "lz4frame_static.h"
 
-static void decompress_data(LZ4F_dctx *dctx, void *src, void *dst,
+static void decompress(LZ4F_dctx *dctx, void *src, void *dst,
                            size_t dstCapacity, size_t readSize) {
   size_t ret = 1;
   const void *srcPtr = (const char *)src;
@@ -40,9 +40,8 @@ static void decompress_data(LZ4F_dctx *dctx, void *src, void *dst,
   }
 }
 
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
-  LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer);
+static void compress_round_trip(const uint8_t* data, size_t size,
+                                FUZZ_dataProducer_t *producer, LZ4F_preferences_t const prefs) {
   size = FUZZ_dataProducer_remainingBytes(producer);
 
   uint8_t *uncompressedData = malloc(size);
@@ -103,7 +102,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
   FUZZ_ASSERT(dctx);
 
-  decompress_data(dctx, dst, rt, rtCapacity, compressedSize);
+  decompress(dctx, dst, rt, rtCapacity, compressedSize);
 
   LZ4F_freeDecompressionContext(dctx);
 
@@ -123,5 +122,25 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   FUZZ_dataProducer_free(producer);
   FUZZ_dataProducer_free(uncompressedProducer);
   LZ4F_freeCompressionContext(ctx);
+}
+
+static void compress_linked_block_mode(const uint8_t* data, size_t size) {
+  FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+  LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer);
+  prefs.frameInfo.blockMode = LZ4F_blockLinked;
+  compress_round_trip(data, size, producer, prefs);
+}
+
+static void compress_independent_block_mode(const uint8_t* data, size_t size) {
+  FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
+  LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer);
+  prefs.frameInfo.blockMode = LZ4F_blockIndependent;
+  compress_round_trip(data, size, producer, prefs);
+}
+
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  compress_linked_block_mode(data, size);
+  compress_independent_block_mode(data, size);
   return 0;
 }
-- 
cgit v0.12


From e595150bafff318f6f66e4b364193b32701449e3 Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Mon, 4 Jul 2022 08:51:37 +0200
Subject: lz4frame: correct start and size after flush

when the block mode changes a flush is executed, to prevent
mixing compressed and uncompressed data.
Prior to this commit dstStart, dstPtr, dstCapacity
where not updated to include the offset from bytesWritten.
For inputs > blockSize this meant the flushed data was
overwritten.

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 lib/lz4frame.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 251521e..f32ed1d 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -886,7 +886,7 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
     size_t const blockSize = cctxPtr->maxBlockSize;
     const BYTE* srcPtr = (const BYTE*)srcBuffer;
     const BYTE* const srcEnd = srcPtr + srcSize;
-    BYTE* const dstStart = (BYTE*)dstBuffer;
+    BYTE* dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
@@ -896,6 +896,9 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
     /* flush currently written block, to continue with new block compression */
     if (cctxPtr->blockCompression != blockCompression) {
       bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+      dstStart = (BYTE*)dstBuffer + bytesWritten;
+      dstPtr = dstStart;
+      dstCapacity -= bytesWritten;
       cctxPtr->blockCompression = blockCompression;
     }
 
-- 
cgit v0.12


From 42eb47d42f041054140b8e08ffc6ba85e9f092f2 Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Tue, 5 Jul 2022 11:35:58 +0200
Subject: uncompressed-api: allow uncompressed_update only for independent
 blocks

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 doc/lz4_manual.html |  12 ----
 lib/lz4.c           |  24 ++------
 lib/lz4.h           |  11 ----
 lib/lz4frame.c      | 172 ++++++++++++++++++++++------------------------------
 lib/lz4frame.h      |   1 +
 lib/lz4hc.c         |  27 ++-------
 6 files changed, 84 insertions(+), 163 deletions(-)

diff --git a/doc/lz4_manual.html b/doc/lz4_manual.html
index 700cb84..037cfc0 100644
--- a/doc/lz4_manual.html
+++ b/doc/lz4_manual.html
@@ -391,18 +391,6 @@ int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
  
 </p></pre><BR>
 
-<pre><b>LZ4LIB_STATIC_API int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize);
-</b><p> Get the size of the dictionary. This can be used for adding data without
- compression to the LZ4 archive. If linked blocked mode is used the memory
- of the dictionary is kept free.
- This way uncompressed data does not influence the effectiveness of the
- dictionary.
- @param LZ4_dict Pointer to the dictionary to get the size of.
- @param dictSize The maximum dictionary size. (Normally 64 KB).
- @return The size of the dictionary.
- 
-</p></pre><BR>
-
 <pre><b></b><p>
  It's possible to have input and output sharing the same buffer,
  for highly constrained memory environments.
diff --git a/lib/lz4.c b/lib/lz4.c
index 289162d..a2272cf 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1679,25 +1679,6 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     return result;
 }
 
-/*! LZ4_getDictSize():
- * Get the size of the dictionary. This can be used for adding data without
- * compression to the LZ4 archive. If linked blocked mode is used the memory
- * of the dictionary is kept free.
- * This way uncompressed data does not influence the effectiveness of the
- * dictionary.
- * @param LZ4_dict Pointer to the dictionary to get the size of.
- * @param dictSize The maximum dictionary size. (Normally 64 KB).
- * @return The size of the dictionary.
- */
-int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize)
-{
-  const LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
-
-  if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
-  if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
-
-  return dictSize;
-}
 
 /*! LZ4_saveDict() :
  *  If previously compressed data block is not guaranteed to remain available at its memory location,
@@ -1709,9 +1690,12 @@ int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize)
 int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
 {
     LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
-    dictSize = LZ4_getDictSize(LZ4_dict, dictSize);
+
     DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);
 
+    if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
+    if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
+
     if (safeBuffer == NULL) assert(dictSize == 0);
     if (dictSize > 0) {
         const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
diff --git a/lib/lz4.h b/lib/lz4.h
index ce2288e..6c068c6 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -509,17 +509,6 @@ LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const c
  */
 LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
 
-/*! LZ4_getDictSize():
- * Get the size of the dictionary. This can be used for adding data without
- * compression to the LZ4 archive. If linked blocked mode is used the memory
- * of the dictionary is kept free.
- * This way uncompressed data does not influence the effectiveness of the
- * dictionary.
- * @param LZ4_dict Pointer to the dictionary to get the size of.
- * @param dictSize The maximum dictionary size. (Normally 64 KB).
- * @return The size of the dictionary.
- */
-LZ4LIB_STATIC_API int LZ4_getDictSize (const LZ4_stream_t* LZ4_dict, int dictSize);
 
 /*! In-place compression and decompression
  *
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index f32ed1d..3042d6f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -841,23 +841,12 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
     return LZ4F_compressBlockHC_continue;
 }
 
-static int LZ4F_maxDictSize(void) {
-  return 64 KB;
-}
-
 /* Save history (up to 64KB) into @tmpBuff */
 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
 {
     if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-        return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
-    return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
-}
-
-static int LZ4F_localDictSize(LZ4F_cctx_t* cctxPtr)
-{
-  if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-    return LZ4_getDictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
-  return LZ4_getDictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
+        return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
+    return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
 }
 
 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
@@ -886,69 +875,64 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
     size_t const blockSize = cctxPtr->maxBlockSize;
     const BYTE* srcPtr = (const BYTE*)srcBuffer;
     const BYTE* const srcEnd = srcPtr + srcSize;
-    BYTE* dstStart = (BYTE*)dstBuffer;
+    BYTE* const dstStart = (BYTE*)dstBuffer;
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
-    size_t bytesWritten = 0;
+    size_t bytesWritten;
     DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
 
-    /* flush currently written block, to continue with new block compression */
-    if (cctxPtr->blockCompression != blockCompression) {
-      bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
-      dstStart = (BYTE*)dstBuffer + bytesWritten;
-      dstPtr = dstStart;
-      dstCapacity -= bytesWritten;
-      cctxPtr->blockCompression = blockCompression;
-    }
-
     RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);   /* state must be initialized and waiting for next block */
-
-    if (blockCompression == LZ4B_COMPRESSED &&
-        dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
-      RETURN_ERROR(dstMaxSize_tooSmall);
+    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
+        RETURN_ERROR(dstMaxSize_tooSmall);
 
     if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
-      RETURN_ERROR(dstMaxSize_tooSmall);
+        RETURN_ERROR(dstMaxSize_tooSmall);
+
+    /* flush currently written block, to continue with new block compression */
+    if (cctxPtr->blockCompression != blockCompression) {
+        bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+        dstPtr += bytesWritten;
+        cctxPtr->blockCompression = blockCompression;
+    }
 
     if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
 
     /* complete tmp buffer */
     if (cctxPtr->tmpInSize > 0) {   /* some data already within tmp buffer */
-      size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
-      assert(blockSize > cctxPtr->tmpInSize);
-      if (sizeToCopy > srcSize) {
-        /* add src to tmpIn buffer */
-        memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
-        srcPtr = srcEnd;
-        cctxPtr->tmpInSize += srcSize;
-        /* still needs some CRC */
-      } else {
-        /* complete tmpIn block and then compress it */
-        lastBlockCompressed = fromTmpBuffer;
-        memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
-        srcPtr += sizeToCopy;
+        size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
+        assert(blockSize > cctxPtr->tmpInSize);
+        if (sizeToCopy > srcSize) {
+            /* add src to tmpIn buffer */
+            memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
+            srcPtr = srcEnd;
+            cctxPtr->tmpInSize += srcSize;
+            /* still needs some CRC */
+        } else {
+            /* complete tmpIn block and then compress it */
+            lastBlockCompressed = fromTmpBuffer;
+            memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
+            srcPtr += sizeToCopy;
+
+            dstPtr += LZ4F_makeBlock(dstPtr,
+                                     cctxPtr->tmpIn, blockSize,
+                                     compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+                                     cctxPtr->cdict,
+                                     cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
+            if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
+            cctxPtr->tmpInSize = 0;
+    }   }
 
+    while ((size_t)(srcEnd - srcPtr) >= blockSize) {
+        /* compress full blocks */
+        lastBlockCompressed = fromSrcBuffer;
         dstPtr += LZ4F_makeBlock(dstPtr,
-                                 cctxPtr->tmpIn, blockSize,
+                                 srcPtr, blockSize,
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                  cctxPtr->cdict,
-                                 cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
-
-        if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
-        cctxPtr->tmpInSize = 0;
-      }
-    }
-
-    while ((size_t)(srcEnd - srcPtr) >= blockSize) {
-      /* compress full blocks */
-      lastBlockCompressed = fromSrcBuffer;
-      dstPtr += LZ4F_makeBlock(dstPtr,
-                               srcPtr, blockSize,
-                               compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
-                               cctxPtr->cdict,
-                               cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
-      srcPtr += blockSize;
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag,
+                                 blockCompression);
+        srcPtr += blockSize;
     }
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
@@ -960,27 +944,20 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
                                  cctxPtr->cdict,
                                  cctxPtr->prefs.frameInfo.blockChecksumFlag,
                                  blockCompression);
-      srcPtr = srcEnd;
+        srcPtr = srcEnd;
     }
 
     /* preserve dictionary within @tmpBuff whenever necessary */
     if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
-      if (compressOptionsPtr->stableSrc) {
-        cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
-      } else {
-        int realDictSize;
-        if (blockCompression == LZ4B_COMPRESSED) {
-          realDictSize = LZ4F_localSaveDict(cctxPtr);
+        /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */
+        assert(blockCompression == LZ4B_COMPRESSED);
+        if (compressOptionsPtr->stableSrc) {
+            cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
         } else {
-          /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate
-           * this is only relevant if linked block mode
-           * */
-          realDictSize = LZ4F_localDictSize(cctxPtr);
+            int const realDictSize = LZ4F_localSaveDict(cctxPtr);
+            assert(0 <= realDictSize && realDictSize <= 64 KB);
+            cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
         }
-
-        assert(0 <= realDictSize && realDictSize <= 64 KB);
-        cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
-      }
     }
 
     /* keep tmpIn within limits */
@@ -989,30 +966,24 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
     {
         /* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
          * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
-        int realDictSize;
-        if (blockCompression == LZ4B_COMPRESSED) {
-          realDictSize = LZ4F_localSaveDict(cctxPtr);
-        } else {
-          /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate*/
-          realDictSize = LZ4F_maxDictSize();
-        }
+        int const realDictSize = LZ4F_localSaveDict(cctxPtr);
         cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
         assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
     }
 
     /* some input data left, necessarily < blockSize */
     if (srcPtr < srcEnd) {
-       /* fill tmp buffer */
-       size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
-       memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
-       cctxPtr->tmpInSize = sizeToCopy;
+        /* fill tmp buffer */
+        size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
+        memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
+        cctxPtr->tmpInSize = sizeToCopy;
     }
 
     if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
-      (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
+        (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
 
     cctxPtr->totalInSize += srcSize;
-    return bytesWritten + (size_t)(dstPtr - dstStart);
+    return (size_t)(dstPtr - dstStart);
 }
 
 /*! LZ4F_compressUpdate() :
@@ -1032,10 +1003,10 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
                      const void* srcBuffer, size_t srcSize,
                      const LZ4F_compressOptions_t* compressOptionsPtr)
 {
-   return LZ4F_compressUpdateImpl(cctxPtr,
-                                 dstBuffer, dstCapacity,
-                                 srcBuffer, srcSize,
-                                 compressOptionsPtr, LZ4B_COMPRESSED);
+     return LZ4F_compressUpdateImpl(cctxPtr,
+                                   dstBuffer, dstCapacity,
+                                   srcBuffer, srcSize,
+                                   compressOptionsPtr, LZ4B_COMPRESSED);
 }
 
 /*! LZ4F_compressUpdate() :
@@ -1044,6 +1015,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
  *  src data is either buffered or compressed into @dstBuffer.
  *  If previously an uncompressed block was written, buffered data is flushed
  *  before appending compressed data is continued.
+ *  This is only supported when LZ4F_blockIndependent is used
  * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
  * @compressOptionsPtr is optional : provide NULL to mean "default".
  * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
@@ -1051,13 +1023,14 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
  *  After an error, the state is left in a UB state, and must be re-initialized.
  */
 size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
-                                               void* dstBuffer, size_t dstCapacity,
-                                               const void* srcBuffer, size_t srcSize,
-                                               const LZ4F_compressOptions_t* compressOptionsPtr) {
-  return LZ4F_compressUpdateImpl(cctxPtr,
-                                 dstBuffer, dstCapacity,
-                                 srcBuffer, srcSize,
-                                 compressOptionsPtr, LZ4B_UNCOMPRESSED);
+                               void* dstBuffer, size_t dstCapacity,
+                         const void* srcBuffer, size_t srcSize,
+                         const LZ4F_compressOptions_t* compressOptionsPtr) {
+    assert(cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockIndependent);
+    return LZ4F_compressUpdateImpl(cctxPtr,
+                                   dstBuffer, dstCapacity,
+                                   srcBuffer, srcSize,
+                                   compressOptionsPtr, LZ4B_UNCOMPRESSED);
 }
 
 
@@ -1090,7 +1063,8 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
                              cctxPtr->tmpIn, cctxPtr->tmpInSize,
                              compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                              cctxPtr->cdict,
-                             cctxPtr->prefs.frameInfo.blockChecksumFlag, cctxPtr->blockCompression);
+                             cctxPtr->prefs.frameInfo.blockChecksumFlag,
+                             cctxPtr->blockCompression);
     assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));
 
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 7d81fa0..691ad50 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -552,6 +552,7 @@ LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);
  *  After an error, the state is left in a UB state, and must be re-initialized or freed.
  *  If previously a compressed block was written, buffered data is flushed
  *  before appending uncompressed data is continued.
+ *  This is only supported when LZ4F_blockIndependent is used
  * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index b5bc880..99650a6 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1154,26 +1154,6 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
     return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);
 }
 
-/*! LZ4_getDictHCSize():
- * Get the size of the dictionary. This can be used for adding data without
- * compression to the LZ4 archive. If linked blocked mode is used the memory
- * of the dictionary is kept free.
- * This way uncompressed data does not influence the effectiveness of the
- * dictionary.
- * @param LZ4_dict Pointer to the dictionary to get the size of.
- * @param dictSize The maximum dictionary size. (Normally 64 KB).
- * @return The size of the dictionary.
- */
-int LZ4_getDictHCSize(const LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
-  const LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
-  int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
-  DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
-  assert(prefixSize >= 0);
-  if (dictSize > 64 KB) dictSize = 64 KB;
-  if (dictSize < 4) dictSize = 0;
-  if (dictSize > prefixSize) dictSize = prefixSize;
-  return dictSize;
-}
 
 
 /* LZ4_saveDictHC :
@@ -1184,7 +1164,12 @@ int LZ4_getDictHCSize(const LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
 int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
 {
     LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
-    dictSize = LZ4_getDictHCSize(LZ4_streamHCPtr, dictSize);
+    int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+    DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
+    assert(prefixSize >= 0);
+    if (dictSize > 64 KB) dictSize = 64 KB;
+    if (dictSize < 4) dictSize = 0;
+    if (dictSize > prefixSize) dictSize = prefixSize;
     if (safeBuffer == NULL) assert(dictSize == 0);
     if (dictSize > 0)
         memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
-- 
cgit v0.12


From 0ac3c74de1b6de584c361f3e9485dde35f10c756 Mon Sep 17 00:00:00 2001
From: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
Date: Tue, 5 Jul 2022 20:52:50 +0200
Subject: review: fix findings

* replace assert with test for LZ4F_uncompressedUpdate
* update documentation to incldue correct docstring
* remove unecessary entry point
* remove compress_linked_block_mode from fuzzing test

Signed-off-by: Alexander Mohr <alexander.m.mohr@mercedes-benz.com>
---
 contrib/meson/meson/examples/meson.build       |  2 +-
 contrib/meson/meson/lib/meson.build            |  2 +-
 doc/lz4frame_manual.html                       |  1 +
 lib/lz4frame.c                                 |  2 +-
 lib/lz4hc.h                                    | 12 ------------
 ossfuzz/round_trip_frame_uncompressed_fuzzer.c |  8 --------
 6 files changed, 4 insertions(+), 23 deletions(-)

diff --git a/contrib/meson/meson/examples/meson.build b/contrib/meson/meson/examples/meson.build
index f6b6c41..65f54ca 100644
--- a/contrib/meson/meson/examples/meson.build
+++ b/contrib/meson/meson/examples/meson.build
@@ -26,7 +26,7 @@ foreach e, src : examples
   executable(
     e,
     lz4_source_root / 'examples' / src,
-    dependencies: [liblz4_dep, liblz4_internal_dep],
+    dependencies: [liblz4_internal_dep],
     install: false
   )
 endforeach
diff --git a/contrib/meson/meson/lib/meson.build b/contrib/meson/meson/lib/meson.build
index 2ff294d..469cd09 100644
--- a/contrib/meson/meson/lib/meson.build
+++ b/contrib/meson/meson/lib/meson.build
@@ -43,7 +43,7 @@ liblz4_dep = declare_dependency(
   include_directories: include_directories(lz4_source_root / 'lib')
 )
 
-if get_option('tests') or get_option('programs')
+if get_option('tests') or get_option('programs') or get_option('examples')
   liblz4_internal = static_library(
     'lz4-internal',
     objects: liblz4.extract_all_objects(recursive: true),
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index c55c6e9..0ae5150 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -355,6 +355,7 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
   After an error, the state is left in a UB state, and must be re-initialized or freed.
   If previously a compressed block was written, buffered data is flushed
   before appending uncompressed data is continued.
+  This is only supported when LZ4F_blockIndependent is used
  `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
            or an error code if it fails (which can be tested using LZ4F_isError())
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 3042d6f..80e244b 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -1026,7 +1026,7 @@ size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
                                void* dstBuffer, size_t dstCapacity,
                          const void* srcBuffer, size_t srcSize,
                          const LZ4F_compressOptions_t* compressOptionsPtr) {
-    assert(cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockIndependent);
+    RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid);
     return LZ4F_compressUpdateImpl(cctxPtr,
                                    dstBuffer, dstCapacity,
                                    srcBuffer, srcSize,
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index 3b31624..f4afc9b 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -405,18 +405,6 @@ LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary(
           LZ4_streamHC_t *working_stream,
     const LZ4_streamHC_t *dictionary_stream);
 
-/*! LZ4_getDictHCSize():
- * Get the size of the dictionary. This can be used for adding data without
- * compression to the LZ4 archive. If linked blocked mode is used the memory
- * of the dictionary is kept free.
- * This way uncompressed data does not influence the effectiveness of the
- * dictionary.
- * @param LZ4_dict Pointer to the dictionary to get the size of.
- * @param dictSize The maximum dictionary size. (Normally 64 KB).
- * @return The size of the dictionary.
- */
-LZ4LIB_STATIC_API int LZ4_getDictHCSize(const LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
-
 #if defined (__cplusplus)
 }
 #endif
diff --git a/ossfuzz/round_trip_frame_uncompressed_fuzzer.c b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
index cf9cbb9..e578052 100644
--- a/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
+++ b/ossfuzz/round_trip_frame_uncompressed_fuzzer.c
@@ -124,13 +124,6 @@ static void compress_round_trip(const uint8_t* data, size_t size,
   LZ4F_freeCompressionContext(ctx);
 }
 
-static void compress_linked_block_mode(const uint8_t* data, size_t size) {
-  FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
-  LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer);
-  prefs.frameInfo.blockMode = LZ4F_blockLinked;
-  compress_round_trip(data, size, producer, prefs);
-}
-
 static void compress_independent_block_mode(const uint8_t* data, size_t size) {
   FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
   LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer);
@@ -140,7 +133,6 @@ static void compress_independent_block_mode(const uint8_t* data, size_t size) {
 
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  compress_linked_block_mode(data, size);
   compress_independent_block_mode(data, size);
   return 0;
 }
-- 
cgit v0.12