summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/lz4frame_manual.html5
-rw-r--r--examples/frameCompress.c133
-rw-r--r--lib/lz4.c14
-rw-r--r--lib/lz4.h2
-rw-r--r--lib/lz4frame.c228
-rw-r--r--lib/lz4frame.h23
-rw-r--r--lib/lz4hc.c17
-rw-r--r--lib/lz4hc.h2
8 files changed, 329 insertions, 95 deletions
diff --git a/doc/lz4frame_manual.html b/doc/lz4frame_manual.html
index 0040c98..9490b51 100644
--- a/doc/lz4frame_manual.html
+++ b/doc/lz4frame_manual.html
@@ -75,6 +75,11 @@
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
</b></pre><BR>
+<pre><b>typedef enum {
+ LZ4B_COMPRESSED,
+ LZ4B_UNCOMPRESSED,
+} LZ4F_blockCompression_t;
+</b></pre><BR>
<pre><b>typedef struct {
LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB; 0 == default */<b>
LZ4F_blockMode_t blockMode; </b>/* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */<b>
diff --git a/examples/frameCompress.c b/examples/frameCompress.c
index 9eaa4da..3219014 100644
--- a/examples/frameCompress.c
+++ b/examples/frameCompress.c
@@ -11,9 +11,9 @@
#include <errno.h>
#include <assert.h>
+#include <getopt.h>
#include <lz4frame.h>
-
#define IN_CHUNK_SIZE (16*1024)
static const LZ4F_preferences_t kPrefs = {
@@ -57,10 +57,11 @@ static compressResult_t
compress_file_internal(FILE* f_in, FILE* f_out,
LZ4F_compressionContext_t ctx,
void* inBuff, size_t inChunkSize,
- void* outBuff, size_t outCapacity)
+ void* outBuff, size_t outCapacity,
+ FILE* f_unc, long uncOffset)
{
compressResult_t result = { 1, 0, 0 }; /* result for an error */
- unsigned long long count_in = 0, count_out;
+ long long count_in = 0, count_out, bytesToOffset = -1;
assert(f_in != NULL); assert(f_out != NULL);
assert(ctx != NULL);
@@ -81,22 +82,48 @@ compress_file_internal(FILE* f_in, FILE* f_out,
/* stream file */
for (;;) {
- size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
+ size_t compressedSize;
+ long long inSize = IN_CHUNK_SIZE;
+ if (uncOffset >= 0) {
+ bytesToOffset = uncOffset - count_in;
+
+ /* read only remaining bytes to offset position */
+ if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) {
+ inSize = bytesToOffset;
+ }
+ }
+
+ /* input data is at uncompressed data offset */
+ if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) {
+ size_t const readSize = fread(inBuff, 1, inSize, f_unc);
+ if (readSize == 0) {
+ uncOffset = -1;
+ continue;
+ }
+ count_in += readSize;
+ compressedSize = LZ4F_uncompressedUpdate(ctx,
+ outBuff, outCapacity,
+ inBuff, readSize,
+ NULL);
+ } else {
+ size_t const readSize = fread(inBuff, 1, inSize, f_in);
if (readSize == 0) break; /* nothing left to read from input file */
count_in += readSize;
-
- size_t const compressedSize = LZ4F_compressUpdate(ctx,
+ compressedSize = LZ4F_compressUpdate(ctx,
outBuff, outCapacity,
inBuff, readSize,
NULL);
- if (LZ4F_isError(compressedSize)) {
- printf("Compression failed: error %u \n", (unsigned)compressedSize);
- return result;
- }
- printf("Writing %u bytes\n", (unsigned)compressedSize);
- safe_fwrite(outBuff, 1, compressedSize, f_out);
- count_out += compressedSize;
+ }
+
+ if (LZ4F_isError(compressedSize)) {
+ printf("Compression failed: error %u \n", (unsigned)compressedSize);
+ return result;
+ }
+
+ printf("Writing %u bytes\n", (unsigned)compressedSize);
+ safe_fwrite(outBuff, 1, compressedSize, f_out);
+ count_out += compressedSize;
}
/* flush whatever remains within internal buffers */
@@ -120,7 +147,8 @@ compress_file_internal(FILE* f_in, FILE* f_out,
}
static compressResult_t
-compress_file(FILE* f_in, FILE* f_out)
+compress_file(FILE* f_in, FILE* f_out,
+ FILE* f_unc, int uncOffset)
{
assert(f_in != NULL);
assert(f_out != NULL);
@@ -137,7 +165,8 @@ compress_file(FILE* f_in, FILE* f_out)
result = compress_file_internal(f_in, f_out,
ctx,
src, IN_CHUNK_SIZE,
- outbuff, outbufCapacity);
+ outbuff, outbufCapacity,
+ f_unc, uncOffset);
} else {
printf("error : resource allocation failed \n");
}
@@ -305,52 +334,106 @@ static int decompress_file(FILE* f_in, FILE* f_out)
}
-int compareFiles(FILE* fp0, FILE* fp1)
+int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset)
{
int result = 0;
+ long bytesRead = 0;
+ long bytesToOffset = -1;
+ long b1Size = 1024;
while (result==0) {
+ char b1[b1Size];
+ size_t r1;
+ size_t bytesToRead = sizeof b1;
+ if (uncOffset >= 0) {
+ bytesToOffset = uncOffset - bytesRead;
+
+ /* read remainder to offset */
+ if (bytesToOffset < b1Size) {
+ bytesToRead = bytesToOffset;
+ }
+ }
+
char b0[1024];
- char b1[1024];
- size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
- size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
+ size_t r0;
+ if (bytesToOffset <= 0 && fpUnc) {
+ bytesToRead = sizeof b1;
+ r0 = fread(b0, 1,bytesToRead, fpUnc);
+ } else {
+ r0 = fread(b0, 1, bytesToRead, fp0);
+ }
+
+ r1 = fread(b1, 1, r0, fp1);
result = (r0 != r1);
if (!r0 || !r1) break;
if (!result) result = memcmp(b0, b1, r0);
+
+ bytesRead += r1;
}
return result;
}
-int main(int argc, const char **argv) {
+int main(int argc, char **argv) {
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
char decFilename[256] = { 0 };
+ int uncOffset = -1;
+ char uncFilename[256] = { 0 };
+ int opt;
+
if (argc < 2) {
printf("Please specify input filename\n");
- return 0;
+ return EXIT_FAILURE;
}
snprintf(inpFilename, 256, "%s", argv[1]);
snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
+ while ((opt = getopt(argc, argv, "o:d:")) != -1) {
+ switch (opt) {
+ case 'd':
+ snprintf(uncFilename, 256, "%s", optarg);
+ break;
+ case 'o':
+ uncOffset = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]);
+ printf("-o uncompressed data offset\n");
+ printf(" inject uncompressed data at this offset into the lz4 file\n");
+ printf("-d uncompressed file\n");
+ printf(" file to inject without compression into the lz4 file\n");
+ return EXIT_FAILURE;
+ }
+ }
+
printf("inp = [%s]\n", inpFilename);
printf("lz4 = [%s]\n", lz4Filename);
printf("dec = [%s]\n", decFilename);
+ if (uncOffset > 0) {
+ printf("unc = [%s]\n", uncFilename);
+ printf("ofs = [%i]\n", uncOffset);
+ }
/* compress */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
+ FILE* const uncFp = fopen(uncFilename, "rb");
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
- compressResult_t const ret = compress_file(inpFp, outFp);
+ compressResult_t const ret = compress_file(
+ inpFp, outFp,
+ uncFp, uncOffset);
fclose(outFp);
fclose(inpFp);
+ if (uncFp)
+ fclose(uncFp);
if (ret.error) {
printf("compress : failed with code %i\n", ret.error);
@@ -383,12 +466,16 @@ int main(int argc, const char **argv) {
/* verify */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const decFp = fopen(decFilename, "rb");
+ FILE* const uncFp = fopen(uncFilename, "rb");
printf("verify : %s <-> %s\n", inpFilename, decFilename);
- int const cmp = compareFiles(inpFp, decFp);
+ int const cmp = compareFiles(inpFp, decFp,
+ uncFp, uncOffset);
fclose(decFp);
fclose(inpFp);
+ if (uncFp)
+ fclose(uncFp);
if (cmp) {
printf("corruption detected : decompressed file differs from original\n");
diff --git a/lib/lz4.c b/lib/lz4.c
index a2272cf..16ed3d3 100644
--- a/lib/lz4.c
+++ b/lib/lz4.c
@@ -1679,6 +1679,15 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
return result;
}
+int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize)
+{
+ LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
+
+ if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
+ if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
+
+ return dictSize;
+}
/*! LZ4_saveDict() :
* If previously compressed data block is not guaranteed to remain available at its memory location,
@@ -1690,12 +1699,9 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char*
int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
{
LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
-
+ dictSize = LZ4_DictSize(LZ4_dict, dictSize);
DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer);
- if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */
- if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; }
-
if (safeBuffer == NULL) assert(dictSize == 0);
if (dictSize > 0) {
const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize;
diff --git a/lib/lz4.h b/lib/lz4.h
index 6c068c6..1e793fd 100644
--- a/lib/lz4.h
+++ b/lib/lz4.h
@@ -346,6 +346,8 @@ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, in
*/
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
+LZ4LIB_API int LZ4_DictSize (LZ4_stream_t* LZ4_dict, int dictSize);
+
/*! LZ4_saveDict() :
* If last 64KB data cannot be guaranteed to remain available at its current memory location,
* save it into a safer place (char* safeBuffer).
diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index a0275ca..bcf9629 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -129,7 +129,8 @@ static int g_debuglog_enable = 1;
**************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
- typedef uint8_t BYTE;
+#include <stdio.h>
+typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
@@ -236,6 +237,7 @@ typedef struct LZ4F_cctx_s
void* lz4CtxPtr;
U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
+ LZ4F_blockCompression_t blockCompression;
} LZ4F_cctx_t;
@@ -757,14 +759,27 @@ static size_t LZ4F_makeBlock(void* dst,
const void* src, size_t srcSize,
compressFunc_t compress, void* lz4ctx, int level,
const LZ4F_CDict* cdict,
- LZ4F_blockChecksum_t crcFlag)
+ LZ4F_blockChecksum_t crcFlag,
+ LZ4F_blockCompression_t blockCompression)
{
BYTE* const cSizePtr = (BYTE*)dst;
- U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
- (int)(srcSize), (int)(srcSize-1),
- level, cdict);
- if (cSize == 0) { /* compression failed */
- DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+ U32 cSize;
+ if (compress != NULL) {
+ cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
+ (int)(srcSize), (int)(srcSize-1),
+ level, cdict);
+ } else {
+ cSize = (U32)srcSize;
+ /* force no compression if compress callback is null */
+ blockCompression = LZ4B_UNCOMPRESSED;
+ }
+
+ if (cSize == 0) { /* compression failed */
+ DEBUGLOG(5, "LZ4F_makeBlock: compression failed, creating a raw block (size %u)", (U32)srcSize);
+ blockCompression = LZ4B_UNCOMPRESSED;
+ }
+
+ if (blockCompression == LZ4B_UNCOMPRESSED) {
cSize = (U32)srcSize;
LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
memcpy(cSizePtr+BHSize, src, srcSize);
@@ -824,33 +839,48 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
return LZ4F_compressBlockHC_continue;
}
+static int LZ4F_maxDictSize(void) {
+ return 64 KB;
+}
+
/* Save history (up to 64KB) into @tmpBuff */
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
{
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
- return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
- return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
+ return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
+ return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), LZ4F_maxDictSize());
+}
+
+static int LZ4F_localDictSize(LZ4F_cctx_t* cctxPtr)
+{
+ if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
+ return LZ4_DictSize ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
+ return LZ4_DictHCSize ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), LZ4F_maxDictSize());
}
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
-/*! LZ4F_compressUpdate() :
+
+ /*! LZ4F_compressUpdateImpl() :
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
* When successful, the function always entirely consumes @srcBuffer.
* src data is either buffered or compressed into @dstBuffer.
- * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * If the block compression does not match the compression of the previous block, the old data is flushed
+ * and operations continue with the new compression mode.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on.
* @compressOptionsPtr is optional : provide NULL to mean "default".
* @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
* or an error code if it fails (which can be tested using LZ4F_isError())
* After an error, the state is left in a UB state, and must be re-initialized.
*/
-size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
- void* dstBuffer, size_t dstCapacity,
+static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
const void* srcBuffer, size_t srcSize,
- const LZ4F_compressOptions_t* compressOptionsPtr)
-{
+ const LZ4F_compressOptions_t* compressOptionsPtr,
+ LZ4F_blockCompression_t blockCompression)
+ {
size_t const blockSize = cctxPtr->maxBlockSize;
const BYTE* srcPtr = (const BYTE*)srcBuffer;
const BYTE* const srcEnd = srcPtr + srcSize;
@@ -858,49 +888,62 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
BYTE* dstPtr = dstStart;
LZ4F_lastBlockStatus lastBlockCompressed = notDone;
compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
-
+ size_t bytesWritten = 0;
DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
+ /* flush currently written block, to continue with new block compression */
+ if (cctxPtr->blockCompression != blockCompression) {
+ bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
+ cctxPtr->blockCompression = blockCompression;
+ }
+
RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */
- if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
- RETURN_ERROR(dstMaxSize_tooSmall);
+
+ if (blockCompression == LZ4B_COMPRESSED &&
+ dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
+ RETURN_ERROR(dstMaxSize_tooSmall);
+
+ if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize)
+ RETURN_ERROR(dstMaxSize_tooSmall);
+
if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull;
/* complete tmp buffer */
if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
- size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
- assert(blockSize > cctxPtr->tmpInSize);
- if (sizeToCopy > srcSize) {
- /* add src to tmpIn buffer */
- memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
- srcPtr = srcEnd;
- cctxPtr->tmpInSize += srcSize;
- /* still needs some CRC */
- } else {
- /* complete tmpIn block and then compress it */
- lastBlockCompressed = fromTmpBuffer;
- memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
- srcPtr += sizeToCopy;
-
- dstPtr += LZ4F_makeBlock(dstPtr,
- cctxPtr->tmpIn, blockSize,
- compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
- cctxPtr->cdict,
- cctxPtr->prefs.frameInfo.blockChecksumFlag);
-
- if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
- cctxPtr->tmpInSize = 0;
- } }
+ size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
+ assert(blockSize > cctxPtr->tmpInSize);
+ if (sizeToCopy > srcSize) {
+ /* add src to tmpIn buffer */
+ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
+ srcPtr = srcEnd;
+ cctxPtr->tmpInSize += srcSize;
+ /* still needs some CRC */
+ } else {
+ /* complete tmpIn block and then compress it */
+ lastBlockCompressed = fromTmpBuffer;
+ memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
+ srcPtr += sizeToCopy;
- while ((size_t)(srcEnd - srcPtr) >= blockSize) {
- /* compress full blocks */
- lastBlockCompressed = fromSrcBuffer;
dstPtr += LZ4F_makeBlock(dstPtr,
- srcPtr, blockSize,
+ cctxPtr->tmpIn, blockSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
- cctxPtr->prefs.frameInfo.blockChecksumFlag);
- srcPtr += blockSize;
+ cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
+
+ if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
+ cctxPtr->tmpInSize = 0;
+ }
+ }
+
+ while ((size_t)(srcEnd - srcPtr) >= blockSize) {
+ /* compress full blocks */
+ lastBlockCompressed = fromSrcBuffer;
+ dstPtr += LZ4F_makeBlock(dstPtr,
+ srcPtr, blockSize,
+ compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
+ cctxPtr->cdict,
+ cctxPtr->prefs.frameInfo.blockChecksumFlag, blockCompression);
+ srcPtr += blockSize;
}
if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
@@ -910,19 +953,29 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
srcPtr, (size_t)(srcEnd - srcPtr),
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
- cctxPtr->prefs.frameInfo.blockChecksumFlag);
- srcPtr = srcEnd;
+ cctxPtr->prefs.frameInfo.blockChecksumFlag,
+ blockCompression);
+ srcPtr = srcEnd;
}
/* preserve dictionary within @tmpBuff whenever necessary */
if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
- if (compressOptionsPtr->stableSrc) {
- cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
+ if (compressOptionsPtr->stableSrc) {
+ cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
+ } else {
+ int realDictSize;
+ if (blockCompression == LZ4B_COMPRESSED) {
+ realDictSize = LZ4F_localSaveDict(cctxPtr);
} else {
- int const realDictSize = LZ4F_localSaveDict(cctxPtr);
- assert(0 <= realDictSize && realDictSize <= 64 KB);
- cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+ /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate
+ * this is only relevant if linked block mode
+ * */
+ realDictSize = LZ4F_localDictSize(cctxPtr);
}
+
+ assert(0 <= realDictSize && realDictSize <= 64 KB);
+ cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+ }
}
/* keep tmpIn within limits */
@@ -931,24 +984,75 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
{
/* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
* note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
- int const realDictSize = LZ4F_localSaveDict(cctxPtr);
+ int realDictSize;
+ if (blockCompression == LZ4B_COMPRESSED) {
+ realDictSize = LZ4F_localSaveDict(cctxPtr);
+ } else {
+ /* only keep the space of the dictionary, so dict data is kept for the next compressedUpdate*/
+ realDictSize = LZ4F_maxDictSize();
+ }
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
}
/* some input data left, necessarily < blockSize */
if (srcPtr < srcEnd) {
- /* fill tmp buffer */
- size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
- memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
- cctxPtr->tmpInSize = sizeToCopy;
+ /* fill tmp buffer */
+ size_t const sizeToCopy = (size_t)(srcEnd - srcPtr);
+ memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
+ cctxPtr->tmpInSize = sizeToCopy;
}
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
- (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
+ (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
cctxPtr->totalInSize += srcSize;
- return (size_t)(dstPtr - dstStart);
+ return bytesWritten + (size_t)(dstPtr - dstStart);
+}
+
+/*! LZ4F_compressUpdate() :
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
+ */
+size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr)
+{
+ return LZ4F_compressUpdateImpl(cctxPtr,
+ dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ compressOptionsPtr, LZ4B_COMPRESSED);
+}
+
+/*! LZ4F_compressUpdate() :
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * When successful, the function always entirely consumes @srcBuffer.
+ * src data is either buffered or compressed into @dstBuffer.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
+ * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
+ * @compressOptionsPtr is optional : provide NULL to mean "default".
+ * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ * After an error, the state is left in a UB state, and must be re-initialized.
+ */
+LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* compressOptionsPtr) {
+ return LZ4F_compressUpdateImpl(cctxPtr,
+ dstBuffer, dstCapacity,
+ srcBuffer, srcSize,
+ compressOptionsPtr, LZ4B_UNCOMPRESSED);
}
@@ -981,7 +1085,7 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
cctxPtr->tmpIn, cctxPtr->tmpInSize,
compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
cctxPtr->cdict,
- cctxPtr->prefs.frameInfo.blockChecksumFlag);
+ cctxPtr->prefs.frameInfo.blockChecksumFlag, cctxPtr->blockCompression);
assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity));
if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked)
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 74f19cd..18d33e1 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -160,6 +160,11 @@ typedef enum {
LZ4F_OBSOLETE_ENUM(skippableFrame)
} LZ4F_frameType_t;
+typedef enum {
+ LZ4B_COMPRESSED,
+ LZ4B_UNCOMPRESSED
+} LZ4F_blockCompression_t;
+
#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS
typedef LZ4F_blockSizeID_t blockSizeID_t;
typedef LZ4F_blockMode_t blockMode_t;
@@ -303,6 +308,8 @@ LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t*
* This value is provided by LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
* After an error, the state is left in a UB state, and must be re-initialized or freed.
+ * If previously an uncompressed block was written, buffered data is flushed
+ * before appending compressed data is continued.
* `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError())
@@ -312,6 +319,22 @@ LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx,
const void* srcBuffer, size_t srcSize,
const LZ4F_compressOptions_t* cOptPtr);
+/*! LZ4F_uncompressedUpdate() :
+ * LZ4F_uncompressedUpdate() can be called repetitively to add as much data uncompressed data as necessary.
+ * Important rule: dstCapacity MUST be large enough to store the entire source buffer as
+ * no compression is done for this operation
+ * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode).
+ * After an error, the state is left in a UB state, and must be re-initialized or freed.
+ * If previously a compressed block was written, buffered data is flushed
+ * before appending uncompressed data is continued.
+ * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
+ * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
+ * or an error code if it fails (which can be tested using LZ4F_isError())
+ */
+LZ4FLIB_API size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctx,
+ void* dstBuffer, size_t dstCapacity,
+ const void* srcBuffer, size_t srcSize,
+ const LZ4F_compressOptions_t* cOptPtr);
/*! LZ4F_flush() :
* When data must be generated and sent immediately, without waiting for a block to be completely filled,
* it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx.
diff --git a/lib/lz4hc.c b/lib/lz4hc.c
index 99650a6..da806ef 100644
--- a/lib/lz4hc.c
+++ b/lib/lz4hc.c
@@ -1154,6 +1154,16 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput);
}
+int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize) {
+ LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
+ int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+ DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
+ assert(prefixSize >= 0);
+ if (dictSize > 64 KB) dictSize = 64 KB;
+ if (dictSize < 4) dictSize = 0;
+ if (dictSize > prefixSize) dictSize = prefixSize;
+ return dictSize;
+}
/* LZ4_saveDictHC :
@@ -1164,12 +1174,7 @@ int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const ch
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
- int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
- DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize);
- assert(prefixSize >= 0);
- if (dictSize > 64 KB) dictSize = 64 KB;
- if (dictSize < 4) dictSize = 0;
- if (dictSize > prefixSize) dictSize = prefixSize;
+ dictSize = LZ4_DictHCSize(LZ4_streamHCPtr, dictSize);
if (safeBuffer == NULL) assert(dictSize == 0);
if (dictSize > 0)
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
diff --git a/lib/lz4hc.h b/lib/lz4hc.h
index f4afc9b..11671dc 100644
--- a/lib/lz4hc.h
+++ b/lib/lz4hc.h
@@ -173,6 +173,8 @@ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr
const char* src, char* dst,
int* srcSizePtr, int targetDstSize);
+LZ4LIB_API int LZ4_DictHCSize(LZ4_streamHC_t* LZ4_streamHCPtr, int dictSize);
+
LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
generated by cgit v0.12 at 2024-11-05 02:46:17 (GMT)