From 4aeb5020c35e2464c25eb69d8bf6c7645b8faf20 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 3 Jun 2022 15:38:54 +0200 Subject: frame-api: add method to insert uncomressed data new method `uncompressed_update` allows to insert blocks without compression into the lz4 stream. The usage is documented in the frameCompress example Signed-off-by: Alexander Mohr --- doc/lz4frame_manual.html | 5 ++ examples/frameCompress.c | 133 ++++++++++++++++++++++----- lib/lz4.c | 14 ++- lib/lz4.h | 2 + lib/lz4frame.c | 228 ++++++++++++++++++++++++++++++++++------------- lib/lz4frame.h | 23 +++++ lib/lz4hc.c | 17 ++-- lib/lz4hc.h | 2 + 8 files changed, 329 insertions(+), 95 deletions(-) diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html index 0040c98..9490b51 100644 --- a/doc/lz4frame_manual.html +++ b/doc/lz4frame_manual.html @@ -75,6 +75,11 @@ LZ4F_OBSOLETE_ENUM(skippableFrame) } LZ4F_frameType_t;
+
typedef enum {
+  LZ4B_COMPRESSED,
+  LZ4B_UNCOMPRESSED,
+} LZ4F_blockCompression_t;
+

typedef struct {
   LZ4F_blockSizeID_t     blockSizeID;         /* max64KB, max256KB, max1MB, max4MB; 0 == default */
   LZ4F_blockMode_t       blockMode;           /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 9eaa4da..3219014 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -11,9 +11,9 @@
 #include 
 #include 
 
+#include 
 #include 
 
-
 #define IN_CHUNK_SIZE  (16*1024)
 
 static const LZ4F_preferences_t kPrefs = {
@@ -57,10 +57,11 @@ static compressResult_t
 compress_file_internal(FILE* f_in, FILE* f_out,
                        LZ4F_compressionContext_t ctx,
                        void* inBuff,  size_t inChunkSize,
-                       void* outBuff, size_t outCapacity)
+                       void* outBuff, size_t outCapacity,
+                       FILE* f_unc, long uncOffset)
 {
     compressResult_t result = { 1, 0, 0 };  /* result for an error */
-    unsigned long long count_in = 0, count_out;
+    long long count_in = 0, count_out, bytesToOffset = -1;
 
     assert(f_in != NULL); assert(f_out != NULL);
     assert(ctx != NULL);
@@ -81,22 +82,48 @@ compress_file_internal(FILE* f_in, FILE* f_out,
 
     /* stream file */
     for (;;) {
-        size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
+      size_t compressedSize;
+      long long inSize = IN_CHUNK_SIZE;
+      if (uncOffset >= 0) {
+        bytesToOffset = uncOffset - count_in;
+
+        /* read only remaining bytes to offset position */
+        if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) {
+          inSize = bytesToOffset;
+        }
+      }
+
+      /* input data is at uncompressed data offset */
+      if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) {
+        size_t const readSize = fread(inBuff, 1, inSize, f_unc);
+        if (readSize == 0) {
+          uncOffset = -1;
+          continue;
+        }
+        count_in += readSize;
+        compressedSize = LZ4F_uncompressedUpdate(ctx,
+                                             outBuff, outCapacity,
+                                             inBuff, readSize,
+                                             NULL);
+      } else {
+        size_t const readSize = fread(inBuff, 1, inSize, f_in);
         if (readSize == 0) break; /* nothing left to read from input file */
         count_in += readSize;
-
-        size_t const compressedSize = LZ4F_compressUpdate(ctx,
+        compressedSize = LZ4F_compressUpdate(ctx,
                                                 outBuff, outCapacity,
                                                 inBuff, readSize,
                                                 NULL);
-        if (LZ4F_isError(compressedSize)) {
-            printf("Compression failed: error %u \n", (unsigned)compressedSize);
-            return result;
-        }
 
-        printf("Writing %u bytes\n", (unsigned)compressedSize);
-        safe_fwrite(outBuff, 1, compressedSize, f_out);
-        count_out += compressedSize;
+      }
+
+      if (LZ4F_isError(compressedSize)) {
+        printf("Compression failed: error %u \n", (unsigned)compressedSize);
+        return result;
+      }
+
+      printf("Writing %u bytes\n", (unsigned)compressedSize);
+      safe_fwrite(outBuff, 1, compressedSize, f_out);
+      count_out += compressedSize;
     }
 
     /* flush whatever remains within internal buffers */
@@ -120,7 +147,8 @@ compress_file_internal(FILE* f_in, FILE* f_out,
 }
 
 static compressResult_t
-compress_file(FILE* f_in, FILE* f_out)
+compress_file(FILE* f_in, FILE* f_out,
+              FILE* f_unc, int uncOffset)
 {
     assert(f_in != NULL);
     assert(f_out != NULL);
@@ -137,7 +165,8 @@ compress_file(FILE* f_in, FILE* f_out)
         result = compress_file_internal(f_in, f_out,
                                         ctx,
                                         src, IN_CHUNK_SIZE,
-                                        outbuff, outbufCapacity);
+                                        outbuff, outbufCapacity,
+                                        f_unc, uncOffset);
     } else {
         printf("error : resource allocation failed \n");
     }
@@ -305,52 +334,106 @@ static int decompress_file(FILE* f_in, FILE* f_out)
 }
 
 
-int compareFiles(FILE* fp0, FILE* fp1)
+int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset)
 {
     int result = 0;
+    long bytesRead = 0;
+    long bytesToOffset = -1;
+    long b1Size = 1024;
 
     while (result==0) {
+        char b1[b1Size];
+        size_t r1;
+        size_t bytesToRead = sizeof b1;
+        if (uncOffset >= 0) {
+          bytesToOffset = uncOffset - bytesRead;
+
+          /* read remainder to offset */
+          if (bytesToOffset < b1Size) {
+            bytesToRead = bytesToOffset;
+          }
+        }
+
         char b0[1024];
-        char b1[1024];
-        size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
-        size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
+        size_t r0;
+        if (bytesToOffset <= 0 && fpUnc) {
+          bytesToRead = sizeof b1;
+          r0 = fread(b0, 1,bytesToRead, fpUnc);
+        } else {
+          r0 = fread(b0, 1, bytesToRead, fp0);
+        }
+
+        r1 = fread(b1, 1, r0, fp1);
 
         result = (r0 != r1);
         if (!r0 || !r1) break;
         if (!result) result = memcmp(b0, b1, r0);
+
+        bytesRead += r1;
     }
 
     return result;
 }
 
 
-int main(int argc, const char **argv) {
+int main(int argc, char **argv) {
     char inpFilename[256] = { 0 };
     char lz4Filename[256] = { 0 };
     char decFilename[256] = { 0 };
 
+    int uncOffset = -1;
+    char uncFilename[256] = { 0 };
+    int opt;
+
     if (argc < 2) {
         printf("Please specify input filename\n");
-        return 0;
+        return EXIT_FAILURE;
     }
 
     snprintf(inpFilename, 256, "%s", argv[1]);
     snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
     snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
 
+    while ((opt = getopt(argc, argv, "o:d:")) != -1) {
+      switch (opt) {
+      case 'd':
+        snprintf(uncFilename, 256, "%s", optarg);
+        break;
+      case 'o':
+        uncOffset = atoi(optarg);
+        break;
+      default:
+        printf("usage: %s  [-o  -d ]\n", argv[0]);
+        printf("-o uncompressed data offset\n");
+        printf("   inject uncompressed data at this offset into the lz4 file\n");
+        printf("-d uncompressed file\n");
+        printf("   file to inject without compression into the lz4 file\n");
+        return EXIT_FAILURE;
+      }
+    }
+
     printf("inp = [%s]\n", inpFilename);
     printf("lz4 = [%s]\n", lz4Filename);
     printf("dec = [%s]\n", decFilename);
+    if (uncOffset > 0) {
+      printf("unc = [%s]\n", uncFilename);
+      printf("ofs = [%i]\n", uncOffset);
+    }
 
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
+        FILE* const uncFp = fopen(uncFilename, "rb");
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
-        compressResult_t const ret = compress_file(inpFp, outFp);
+        compressResult_t const ret = compress_file(
+            inpFp, outFp,
+            uncFp, uncOffset);
 
         fclose(outFp);
         fclose(inpFp);
+        if (uncFp)
+          fclose(uncFp);
 
         if (ret.error) {
             printf("compress : failed with code %i\n", ret.error);
@@ -383,12 +466,16 @@ int main(int argc, const char **argv) {
     /* verify */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const decFp = fopen(decFilename, "rb");
+        FILE* const uncFp = fopen(uncFilename, "rb");
 
         printf("verify : %s <-> %s\n", inpFilename, decFilename);
-        int const cmp = compareFiles(inpFp, decFp);
+        int const cmp = compareFiles(inpFp, decFp,
+                                     uncFp, uncOffset);
 
         fclose(decFp);
         fclose(inpFp);
+        if (uncFp)
+          fclose(uncFp);
 
         if (cmp) {
             printf("corruption detected : decompressed file differs from original\n");
diff --git a/lib/lz4.c b/lib/lz4.c
index a2272cf..16ed3d3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1679,6 +1679,15 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
     return result;
 }
 
+int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize)
+{
+  LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
+
+  if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
+  if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
+
+  return dictSize;
+}
 
 /*! LZ4_saveDict() :
  *  If previously compressed data block is not guaranteed to remain available at its memory location,
@@ -1690,12 +1699,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
 int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
 {
     LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
-
+    dictSize = LZ4_DictSize(LZ4_dict, dictSize);
     DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);
 
-    if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
-    if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
-
     if (safeBuffer == NULL) assert(dictSize == 0);
     if (dictSize > 0) {
         const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
diff --git a/lib/lz4.h b/lib/lz4.h
index 6c068c6..1e793fd 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -346,6 +346,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
  */
 LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
 
+LZ4LIB_API int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize);
+
 /*! LZ4_saveDict() :
  *  If last 64KB data cannot be guaranteed to remain available at its current memory location,
  *  save it into a safer place (char* safeBuffer).
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a0275ca..bcf9629 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -129,7 +129,8 @@ static int g_debuglog_enable = 1;
 **************************************/
 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include 
-  typedef  uint8_t BYTE;
+#include 
+typedef  uint8_t BYTE;
   typedef uint16_t U16;
   typedef uint32_t U32;
   typedef  int32_t S32;
@@ -236,6 +237,7 @@ typedef struct LZ4F_cctx_s
     void*  lz4CtxPtr;
     U16    lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
     U16    lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+    LZ4F_blockCompression_t  blockCompression;
 } LZ4F_cctx_t;
 
 
@@ -757,14 +759,27 @@ static size_t LZ4F_makeBlock(void* dst,
                        const void* src, size_t srcSize,
                              compressFunc_t compress, void* lz4ctx, int level,
                        const LZ4F_CDict* cdict,
-                             LZ4F_blockChecksum_t crcFlag)
+                             LZ4F_blockChecksum_t crcFlag,
+                             LZ4F_blockCompression_t blockCompression)
 {
     BYTE* const cSizePtr = (BYTE*)dst;
-    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
-                                      (int)(srcSize), (int)(srcSize-1),
-                                      level, cdict);
-    if (cSize == 0) {  /* compression failed */
-        DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+    U32 cSize;
+    if (compress != NULL) {
+      cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
+                            (int)(srcSize), (int)(srcSize-1),
+                            level, cdict);
+    } else {
+      cSize = (U32)srcSize;
+      /* force no compression if compress callback is null */
+      blockCompression = LZ4B_UNCOMPRESSED;
+    }
+
+    if (cSize == 0) { /* compression failed */
+      DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+      blockCompression = LZ4B_UNCOMPRESSED;
+    }
+
+    if (blockCompression == LZ4B_UNCOMPRESSED) {
         cSize = (U32)srcSize;
         LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
         memcpy(cSizePtr+BHSize, src, srcSize);
@@ -824,33 +839,48 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
     return LZ4F_compressBlockHC_continue;
 }
 
+static int LZ4F_maxDictSize(void) {
+  return 64 KB;
+}
+
 /* Save history (up to 64KB) into @tmpBuff */
 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
 {
     if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-        return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
-    return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
+        return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
+    return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
+}
+
+static int LZ4F_localDictSize(LZ4F_cctx_t* cctxPtr)
+{
+  if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+    return LZ4_DictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
+  return LZ4_DictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
 }
 
 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
 
 static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
 
-/*! LZ4F_compressUpdate() :
+
+ /*! LZ4F_compressUpdateImpl() :
  *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
  *  When successful, the function always entirely consumes @srcBuffer.
  *  src data is either buffered or compressed into @dstBuffer.
- * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ *  If the block compression does not match the compression of the previous block, the old data is flushed
+ *  and operations continue with the new compression mode.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
  * @compressOptionsPtr is optional : provide NULL to mean "default".
  * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  After an error, the state is left in a UB state, and must be re-initialized.
  */
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
-                           void* dstBuffer, size_t dstCapacity,
+static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
+                     void* dstBuffer, size_t dstCapacity,
                      const void* srcBuffer, size_t srcSize,
-                     const LZ4F_compressOptions_t* compressOptionsPtr)
-{
+                     const LZ4F_compressOptions_t* compressOptionsPtr,
+                     LZ4F_blockCompression_t blockCompression)
+  {
     size_t const blockSize = cctxPtr->maxBlockSize;
     const BYTE* srcPtr = (const BYTE*)srcBuffer;
     const BYTE* const srcEnd = srcPtr + srcSize;
@@ -858,49 +888,62 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
-
+    size_t bytesWritten = 0;
     DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
 
+    /* flush currently written block, to continue with new block compression */
+    if (cctxPtr->blockCompression != blockCompression) {
+      bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+      cctxPtr->blockCompression = blockCompression;
+    }
+
     RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);   /* state must be initialized and waiting for next block */
-    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
-        RETURN_ERROR(dstMaxSize_tooSmall);
+
+    if (blockCompression == LZ4B_COMPRESSED &&
+        dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
+      RETURN_ERROR(dstMaxSize_tooSmall);
+
+    if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
+      RETURN_ERROR(dstMaxSize_tooSmall);
+
     if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
 
     /* complete tmp buffer */
     if (cctxPtr->tmpInSize > 0) {   /* some data already within tmp buffer */
-        size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
-        assert(blockSize > cctxPtr->tmpInSize);
-        if (sizeToCopy > srcSize) {
-            /* add src to tmpIn buffer */
-            memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
-            srcPtr = srcEnd;
-            cctxPtr->tmpInSize += srcSize;
-            /* still needs some CRC */
-        } else {
-            /* complete tmpIn block and then compress it */
-            lastBlockCompressed = fromTmpBuffer;
-            memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
-            srcPtr += sizeToCopy;
-
-            dstPtr += LZ4F_makeBlock(dstPtr,
-                                     cctxPtr->tmpIn, blockSize,
-                                     compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
-                                     cctxPtr->cdict,
-                                     cctxPtr->prefs.frameInfo.blockChecksumFlag);
-
-            if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
-            cctxPtr->tmpInSize = 0;
-    }   }
+      size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
+      assert(blockSize > cctxPtr->tmpInSize);
+      if (sizeToCopy > srcSize) {
+        /* add src to tmpIn buffer */
+        memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
+        srcPtr = srcEnd;
+        cctxPtr->tmpInSize += srcSize;
+        /* still needs some CRC */
+      } else {
+        /* complete tmpIn block and then compress it */
+        lastBlockCompressed = fromTmpBuffer;
+        memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
+        srcPtr += sizeToCopy;
 
-    while ((size_t)(srcEnd - srcPtr) >= blockSize) {
-        /* compress full blocks */
-        lastBlockCompressed = fromSrcBuffer;
         dstPtr += LZ4F_makeBlock(dstPtr,
-                                 srcPtr, blockSize,
+                                 cctxPtr->tmpIn, blockSize,
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                  cctxPtr->cdict,
-                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
-        srcPtr += blockSize;
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
+
+        if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
+        cctxPtr->tmpInSize = 0;
+      }
+    }
+
+    while ((size_t)(srcEnd - srcPtr) >= blockSize) {
+      /* compress full blocks */
+      lastBlockCompressed = fromSrcBuffer;
+      dstPtr += LZ4F_makeBlock(dstPtr,
+                               srcPtr, blockSize,
+                               compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+                               cctxPtr->cdict,
+                               cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
+      srcPtr += blockSize;
     }
 
     if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
@@ -910,19 +953,29 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
                                  srcPtr, (size_t)(srcEnd - srcPtr),
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                  cctxPtr->cdict,
-                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
-        srcPtr  = srcEnd;
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag,
+                                 blockCompression);
+      srcPtr = srcEnd;
     }
 
     /* preserve dictionary within @tmpBuff whenever necessary */
     if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
-        if (compressOptionsPtr->stableSrc) {
-            cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
+      if (compressOptionsPtr->stableSrc) {
+        cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
+      } else {
+        int realDictSize;
+        if (blockCompression == LZ4B_COMPRESSED) {
+          realDictSize = LZ4F_localSaveDict(cctxPtr);
         } else {
-            int const realDictSize = LZ4F_localSaveDict(cctxPtr);
-            assert(0 <= realDictSize && realDictSize <= 64 KB);
-            cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+          /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate
+           * this is only relevant if linked block mode
+           * */
+          realDictSize = LZ4F_localDictSize(cctxPtr);
         }
+
+        assert(0 <= realDictSize && realDictSize <= 64 KB);
+        cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+      }
     }
 
     /* keep tmpIn within limits */
@@ -931,24 +984,75 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     {
         /* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
          * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
-        int const realDictSize = LZ4F_localSaveDict(cctxPtr);
+        int realDictSize;
+        if (blockCompression == LZ4B_COMPRESSED) {
+          realDictSize = LZ4F_localSaveDict(cctxPtr);
+        } else {
+          /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate*/
+          realDictSize = LZ4F_maxDictSize();
+        }
         cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
         assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
     }
 
     /* some input data left, necessarily < blockSize */
     if (srcPtr < srcEnd) {
-        /* fill tmp buffer */
-        size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
-        memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
-        cctxPtr->tmpInSize = sizeToCopy;
+       /* fill tmp buffer */
+       size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
+       memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
+       cctxPtr->tmpInSize = sizeToCopy;
     }
 
     if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
-        (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
+      (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
 
     cctxPtr->totalInSize += srcSize;
-    return (size_t)(dstPtr - dstStart);
+    return bytesWritten + (size_t)(dstPtr - dstStart);
+}
+
+/*! LZ4F_compressUpdate() :
+ *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ *  When successful, the function always entirely consumes @srcBuffer.
+ *  src data is either buffered or compressed into @dstBuffer.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ *  After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+                           void* dstBuffer, size_t dstCapacity,
+                     const void* srcBuffer, size_t srcSize,
+                     const LZ4F_compressOptions_t* compressOptionsPtr)
+{
+   return LZ4F_compressUpdateImpl(cctxPtr,
+                                 dstBuffer, dstCapacity,
+                                 srcBuffer, srcSize,
+                                 compressOptionsPtr, LZ4B_COMPRESSED);
+}
+
+/*! LZ4F_compressUpdate() :
+ *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ *  When successful, the function always entirely consumes @srcBuffer.
+ *  src data is either buffered or compressed into @dstBuffer.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ *  After an error, the state is left in a UB state, and must be re-initialized.
+ */
+LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+                                               void* dstBuffer, size_t dstCapacity,
+                                               const void* srcBuffer, size_t srcSize,
+                                               const LZ4F_compressOptions_t* compressOptionsPtr) {
+  return LZ4F_compressUpdateImpl(cctxPtr,
+                                 dstBuffer, dstCapacity,
+                                 srcBuffer, srcSize,
+                                 compressOptionsPtr, LZ4B_UNCOMPRESSED);
 }
 
 
@@ -981,7 +1085,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
                              cctxPtr->tmpIn, cctxPtr->tmpInSize,
                              compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                              cctxPtr->cdict,
-                             cctxPtr->prefs.frameInfo.blockChecksumFlag);
+                             cctxPtr->prefs.frameInfo.blockChecksumFlag, cctxPtr->blockCompression);
     assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));
 
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 74f19cd..18d33e1 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -160,6 +160,11 @@ typedef enum {
     LZ4F_OBSOLETE_ENUM(skippableFrame)
 } LZ4F_frameType_t;
 
+typedef enum {
+  LZ4B_COMPRESSED,
+  LZ4B_UNCOMPRESSED
+} LZ4F_blockCompression_t;
+
 #ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
 typedef LZ4F_blockSizeID_t blockSizeID_t;
 typedef LZ4F_blockMode_t blockMode_t;
@@ -303,6 +308,8 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
  *  This value is provided by LZ4F_compressBound().
  *  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
  *  After an error, the state is left in a UB state, and must be re-initialized or freed.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
  * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
@@ -312,6 +319,22 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
                                  const void* srcBuffer, size_t srcSize,
                                  const LZ4F_compressOptions_t* cOptPtr);
 
+/*! LZ4F_uncompressedUpdate() :
+ *  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ *  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ *  no compression is done for this operation
+ *  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ *  After an error, the state is left in a UB state, and must be re-initialized or freed.
+ *  If previously a compressed block was written, buffered data is flushed
+ *  before appending uncompressed data is continued.
+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+                                       void* dstBuffer, size_t dstCapacity,
+                                       const void* srcBuffer, size_t srcSize,
+                                       const LZ4F_compressOptions_t* cOptPtr);
 /*! LZ4F_flush() :
  *  When data must be generated and sent immediately, without waiting for a block to be completely filled,
  *  it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 99650a6..da806ef 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1154,6 +1154,16 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
     return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);
 }
 
+int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
+  LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
+  int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+  DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
+  assert(prefixSize >= 0);
+  if (dictSize > 64 KB) dictSize = 64 KB;
+  if (dictSize < 4) dictSize = 0;
+  if (dictSize > prefixSize) dictSize = prefixSize;
+  return dictSize;
+}
 
 
 /* LZ4_saveDictHC :
@@ -1164,12 +1174,7 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
 int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
 {
     LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
-    int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
-    DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
-    assert(prefixSize >= 0);
-    if (dictSize > 64 KB) dictSize = 64 KB;
-    if (dictSize < 4) dictSize = 0;
-    if (dictSize > prefixSize) dictSize = prefixSize;
+    dictSize = LZ4_DictHCSize(LZ4_streamHCPtr, dictSize);
     if (safeBuffer == NULL) assert(dictSize == 0);
     if (dictSize > 0)
         memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index f4afc9b..11671dc 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -173,6 +173,8 @@ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr
                                            const char* src, char* dst,
                                                  int* srcSizePtr, int targetDstSize);
 
+LZ4LIB_API int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
+
 LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
 
 
-- 
cgit v0.12