Merge pull request #1094 from alexmohr/add-uncompressed-api

frame-api: add function to insert uncomressed data

2 files changed, 122 insertions, 19 deletions

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a0275ca..80e244b 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -220,6 +220,9 @@ static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE;  /* block footer : checks
 /*-************************************
 *  Structures and local types
 **************************************/
+
+typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_blockCompression_t;
+
 typedef struct LZ4F_cctx_s
 {
     LZ4F_preferences_t prefs;
@@ -236,6 +239,7 @@ typedef struct LZ4F_cctx_s
     void*  lz4CtxPtr;
     U16    lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
     U16    lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+    LZ4F_blockCompression_t  blockCompression;
 } LZ4F_cctx_t;
 
 
@@ -757,14 +761,27 @@ static size_t LZ4F_makeBlock(void* dst,
                        const void* src, size_t srcSize,
                              compressFunc_t compress, void* lz4ctx, int level,
                        const LZ4F_CDict* cdict,
-                             LZ4F_blockChecksum_t crcFlag)
+                             LZ4F_blockChecksum_t crcFlag,
+                             LZ4F_blockCompression_t blockCompression)
 {
     BYTE* const cSizePtr = (BYTE*)dst;
-    U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
-                                      (int)(srcSize), (int)(srcSize-1),
-                                      level, cdict);
-    if (cSize == 0) {  /* compression failed */
-        DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+    U32 cSize;
+    if (compress != NULL) {
+      cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
+                            (int)(srcSize), (int)(srcSize-1),
+                            level, cdict);
+    } else {
+      cSize = (U32)srcSize;
+      /* force no compression if compress callback is null */
+      blockCompression = LZ4B_UNCOMPRESSED;
+    }
+
+    if (cSize == 0) { /* compression failed */
+      DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+      blockCompression = LZ4B_UNCOMPRESSED;
+    }
+
+    if (blockCompression == LZ4B_UNCOMPRESSED) {
         cSize = (U32)srcSize;
         LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
         memcpy(cSizePtr+BHSize, src, srcSize);
@@ -836,21 +853,25 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
 
 static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
 
-/*! LZ4F_compressUpdate() :
+
+ /*! LZ4F_compressUpdateImpl() :
  *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
  *  When successful, the function always entirely consumes @srcBuffer.
  *  src data is either buffered or compressed into @dstBuffer.
- * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ *  If the block compression does not match the compression of the previous block, the old data is flushed
+ *  and operations continue with the new compression mode.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
  * @compressOptionsPtr is optional : provide NULL to mean "default".
  * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
  *           or an error code if it fails (which can be tested using LZ4F_isError())
  *  After an error, the state is left in a UB state, and must be re-initialized.
  */
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
-                           void* dstBuffer, size_t dstCapacity,
+static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
+                     void* dstBuffer, size_t dstCapacity,
                      const void* srcBuffer, size_t srcSize,
-                     const LZ4F_compressOptions_t* compressOptionsPtr)
-{
+                     const LZ4F_compressOptions_t* compressOptionsPtr,
+                     LZ4F_blockCompression_t blockCompression)
+  {
     size_t const blockSize = cctxPtr->maxBlockSize;
     const BYTE* srcPtr = (const BYTE*)srcBuffer;
     const BYTE* const srcEnd = srcPtr + srcSize;
@@ -858,12 +879,23 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     LZ4F_lastBlockStatus lastBlockCompressed = notDone;
     compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
-
+    size_t bytesWritten;
     DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
 
     RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);   /* state must be initialized and waiting for next block */
     if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
         RETURN_ERROR(dstMaxSize_tooSmall);
+
+    if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
+        RETURN_ERROR(dstMaxSize_tooSmall);
+
+    /* flush currently written block, to continue with new block compression */
+    if (cctxPtr->blockCompression != blockCompression) {
+        bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+        dstPtr += bytesWritten;
+        cctxPtr->blockCompression = blockCompression;
+    }
+
     if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
 
     /* complete tmp buffer */
@@ -886,8 +918,7 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
                                      cctxPtr->tmpIn, blockSize,
                                      compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                      cctxPtr->cdict,
-                                     cctxPtr->prefs.frameInfo.blockChecksumFlag);
-
+                                     cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
             if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
             cctxPtr->tmpInSize = 0;
     }   }
@@ -899,7 +930,8 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
                                  srcPtr, blockSize,
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                  cctxPtr->cdict,
-                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag,
+                                 blockCompression);
         srcPtr += blockSize;
     }
 
@@ -910,12 +942,15 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
                                  srcPtr, (size_t)(srcEnd - srcPtr),
                                  compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                                  cctxPtr->cdict,
-                                 cctxPtr->prefs.frameInfo.blockChecksumFlag);
-        srcPtr  = srcEnd;
+                                 cctxPtr->prefs.frameInfo.blockChecksumFlag,
+                                 blockCompression);
+        srcPtr = srcEnd;
     }
 
     /* preserve dictionary within @tmpBuff whenever necessary */
     if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
+        /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */
+        assert(blockCompression == LZ4B_COMPRESSED);
         if (compressOptionsPtr->stableSrc) {
             cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
         } else {
@@ -951,6 +986,53 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
     return (size_t)(dstPtr - dstStart);
 }
 
+/*! LZ4F_compressUpdate() :
+ *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ *  When successful, the function always entirely consumes @srcBuffer.
+ *  src data is either buffered or compressed into @dstBuffer.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ *  After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+                           void* dstBuffer, size_t dstCapacity,
+                     const void* srcBuffer, size_t srcSize,
+                     const LZ4F_compressOptions_t* compressOptionsPtr)
+{
+     return LZ4F_compressUpdateImpl(cctxPtr,
+                                   dstBuffer, dstCapacity,
+                                   srcBuffer, srcSize,
+                                   compressOptionsPtr, LZ4B_COMPRESSED);
+}
+
+/*! LZ4F_compressUpdate() :
+ *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ *  When successful, the function always entirely consumes @srcBuffer.
+ *  src data is either buffered or compressed into @dstBuffer.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
+ *  This is only supported when LZ4F_blockIndependent is used
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ *  After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+                               void* dstBuffer, size_t dstCapacity,
+                         const void* srcBuffer, size_t srcSize,
+                         const LZ4F_compressOptions_t* compressOptionsPtr) {
+    RETURN_ERROR_IF(cctxPtr->prefs.frameInfo.blockMode != LZ4F_blockIndependent, blockMode_invalid);
+    return LZ4F_compressUpdateImpl(cctxPtr,
+                                   dstBuffer, dstCapacity,
+                                   srcBuffer, srcSize,
+                                   compressOptionsPtr, LZ4B_UNCOMPRESSED);
+}
+
 
 /*! LZ4F_flush() :
  *  When compressed data must be sent immediately, without waiting for a block to be filled,
@@ -981,7 +1063,8 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
                              cctxPtr->tmpIn, cctxPtr->tmpInSize,
                              compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
                              cctxPtr->cdict,
-                             cctxPtr->prefs.frameInfo.blockChecksumFlag);
+                             cctxPtr->prefs.frameInfo.blockChecksumFlag,
+                             cctxPtr->blockCompression);
     assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));
 
     if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 6bf20e4..2c5a559 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -306,6 +306,8 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
  *  This value is provided by LZ4F_compressBound().
  *  If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
  *  After an error, the state is left in a UB state, and must be re-initialized or freed.
+ *  If previously an uncompressed block was written, buffered data is flushed
+ *  before appending compressed data is continued.
  * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
  * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
  *           or an error code if it fails (which can be tested using LZ4F_isError())
@@ -545,6 +547,24 @@ LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult);
 
 LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(unsigned);
 
+/*! LZ4F_uncompressedUpdate() :
+ *  LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ *  Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ *  no compression is done for this operation
+ *  If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ *  After an error, the state is left in a UB state, and must be re-initialized or freed.
+ *  If previously a compressed block was written, buffered data is flushed
+ *  before appending uncompressed data is continued.
+ *  This is only supported when LZ4F_blockIndependent is used
+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ *           or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+LZ4FLIB_STATIC_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+                                                  void* dstBuffer, size_t dstCapacity,
+                                                  const void* srcBuffer, size_t srcSize,
+                                                  const LZ4F_compressOptions_t* cOptPtr);
+
 /**********************************
  *  Bulk processing dictionary API
  *********************************/