Added : HC streaming ring buffer example

3 files changed, 263 insertions, 23 deletions

diff --git a/examples/HCStreaming_ringBuffer.c b/examples/HCStreaming_ringBuffer.c
new file mode 100755
index 0000000..3532fa3
--- /dev/null
+++ b/examples/HCStreaming_ringBuffer.c
@@ -0,0 +1,208 @@
+// LZ4 HC streaming API example : ring buffer
+// Based on previous work from Takayuki Matsuoka
+
+
+/**************************************
+ Compiler Options
+**************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  define _CRT_SECURE_NO_WARNINGS // for MSVC
+#  define snprintf sprintf_s
+#endif
+
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#ifdef __GNUC__
+#  pragma GCC diagnostic ignored "-Wmissing-braces"   /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
+#endif
+
+
+
+/**************************************
+   Includes
+**************************************/
+#include "lz4hc.h"
+#include "lz4.h"
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+    MESSAGE_MAX_BYTES   = 1024,
+    RING_BUFFER_BYTES   = 1024 * 256 + MESSAGE_MAX_BYTES,
+    DICT_BYTES          = 65536,
+};
+
+
+size_t write_int32(FILE* fp, int32_t i) {
+    return fwrite(&i, sizeof(i), 1, fp);
+}
+
+size_t write_bin(FILE* fp, const void* array, int arrayBytes) {
+    return fwrite(array, 1, arrayBytes, fp);
+}
+
+size_t read_int32(FILE* fp, int32_t* i) {
+    return fread(i, sizeof(*i), 1, fp);
+}
+
+size_t read_bin(FILE* fp, void* array, int arrayBytes) {
+    return fread(array, 1, arrayBytes, fp);
+}
+
+
+void test_compress(FILE* outFp, FILE* inpFp)
+{
+    LZ4_streamHC_t lz4Stream_body = { 0 };
+    LZ4_streamHC_t* lz4Stream = &lz4Stream_body;
+
+    static char inpBuf[RING_BUFFER_BYTES];
+    int inpOffset = 0;
+
+    for(;;)
+        {
+        // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer.
+        char* const inpPtr = &inpBuf[inpOffset];
+        const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1;
+        const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
+        if (0 == inpBytes) break;
+
+        {
+            char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
+            const int cmpBytes = LZ4_compressHC_continue(lz4Stream, inpPtr, cmpBuf, inpBytes);
+            if(cmpBytes <= 0) break;
+            write_int32(outFp, cmpBytes);
+            write_bin(outFp, cmpBuf, cmpBytes);
+
+            inpOffset += inpBytes;
+
+            // Wraparound the ringbuffer offset
+            if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES)
+                inpOffset = 0;
+        }
+    }
+
+    write_int32(outFp, 0);
+}
+
+
+void test_decompress(FILE* outFp, FILE* inpFp)
+{
+    static char decBuf[RING_BUFFER_BYTES];
+    int decOffset = 0;
+    LZ4_streamDecode_t lz4StreamDecode_body = { 0 };
+    LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body;
+
+    for(;;) {
+        int cmpBytes = 0;
+        char cmpBuf[LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)];
+
+        {
+            const size_t r0 = read_int32(inpFp, &cmpBytes);
+			size_t r1;
+            if(r0 != 1 || cmpBytes <= 0) break;
+
+            r1 = read_bin(inpFp, cmpBuf, cmpBytes);
+            if(r1 != (size_t) cmpBytes) break;
+        }
+
+        {
+            char* const decPtr = &decBuf[decOffset];
+            const int decBytes = LZ4_decompress_safe_continue(
+                lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES);
+            if(decBytes <= 0) 
+				break;
+            decOffset += decBytes;
+            write_bin(outFp, decPtr, decBytes);
+
+            // Wraparound the ringbuffer offset
+            if(decOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) decOffset = 0;
+        }
+    }
+}
+
+
+int compare(FILE* f0, FILE* f1)
+{
+    int result = 0;
+
+    while(0 == result) {
+        char b0[65536];
+        char b1[65536];
+        const size_t r0 = fread(b0, 1, sizeof(b0), f0);
+        const size_t r1 = fread(b1, 1, sizeof(b1), f1);
+
+        result = (int) r0 - (int) r1;
+
+        if(0 == r0 || 0 == r1) {
+            break;
+        }
+        if(0 == result) {
+            result = memcmp(b0, b1, r0);
+        }
+    }
+
+    return result;
+}
+
+
+int main(int argc, char** argv)
+{
+    char inpFilename[256] = { 0 };
+    char lz4Filename[256] = { 0 };
+    char decFilename[256] = { 0 };
+
+    if(argc < 2) {
+        printf("Please specify input filename\n");
+        return 0;
+    }
+
+    snprintf(inpFilename, 256, "%s", argv[1]);
+    snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], 0);
+    snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], 0);
+
+    printf("inp = [%s]\n", inpFilename);
+    printf("lz4 = [%s]\n", lz4Filename);
+    printf("dec = [%s]\n", decFilename);
+
+    // compress
+    {
+        FILE* inpFp = fopen(inpFilename, "rb");
+        FILE* outFp = fopen(lz4Filename, "wb");
+
+        test_compress(outFp, inpFp);
+
+        fclose(outFp);
+        fclose(inpFp);
+    }
+
+    // decompress
+    {
+        FILE* inpFp = fopen(lz4Filename, "rb");
+        FILE* outFp = fopen(decFilename, "wb");
+
+        test_decompress(outFp, inpFp);
+
+        fclose(outFp);
+        fclose(inpFp);
+    }
+
+    // verify
+    {
+        FILE* inpFp = fopen(inpFilename, "rb");
+        FILE* decFp = fopen(decFilename, "rb");
+
+        const int cmp = compare(inpFp, decFp);
+        if(0 == cmp) {
+            printf("Verify : OK\n");
+        } else {
+            printf("Verify : NG\n");
+        }
+
+        fclose(decFp);
+        fclose(inpFp);
+    }
+
+    return 0;
+}
diff --git a/examples/Makefile b/examples/Makefile
index 5704c55..b5f9a7e 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -58,7 +58,7 @@ endif
 
 default: all
 
-all: printVersion doubleBuffer ringBuffer lineCompress
+all: printVersion doubleBuffer ringBuffer ringBufferHC lineCompress
 
 printVersion: $(LZ4DIR)/lz4.c printVersion.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
@@ -69,6 +69,9 @@ doubleBuffer: $(LZ4DIR)/lz4.c blockStreaming_doubleBuffer.c
 ringBuffer  : $(LZ4DIR)/lz4.c blockStreaming_ringBuffer.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
+ringBufferHC: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c HCStreaming_ringBuffer.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
 lineCompress: $(LZ4DIR)/lz4.c blockStreaming_lineByLine.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
@@ -76,10 +79,11 @@ test : all
 	./printVersion$(EXT)
 	./doubleBuffer$(EXT) $(TESTFILE)
 	./ringBuffer$(EXT)   $(TESTFILE)
+	./ringBufferHC$(EXT) $(TESTFILE)
 	./lineCompress$(EXT) $(TESTFILE)
 
 clean:
 	@rm -f core *.o *.dec *-0 *-8192 *.lz4s \
-        printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) lineCompress$(EXT)
+        printVersion$(EXT) doubleBuffer$(EXT) ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT)
 	@echo Cleaning completed
 
diff --git a/lz4hc.c b/lz4hc.c
index 38e0ce4..ce273de 100755
--- a/lz4hc.c
+++ b/lz4hc.c
@@ -379,6 +379,21 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
 }
 
 
+static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
+{
+	if (ctxPtr->end >= ctxPtr->base + 4)
+		LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   // finish referencing dictionary content
+    // Note : need to handle risk of index overflow
+    // Use only one memory segment for dict, so any previous External Dict is lost at this stage
+    ctxPtr->lowLimit  = ctxPtr->dictLimit;
+    ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
+    ctxPtr->dictBase  = ctxPtr->base;
+    ctxPtr->base = newBlock - ctxPtr->dictLimit;
+    ctxPtr->end  = newBlock;
+    ctxPtr->nextToUpdate = ctxPtr->dictLimit;   // reference table must skip to from beginning of block
+}
+
+
 static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const p1Limit)
 {
     const BYTE* const p1Start = p1;
@@ -397,21 +412,6 @@ static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* co
 }
 
 
-static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
-{
-	if (ctxPtr->end >= ctxPtr->base + 4)
-		LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   // finish referencing dictionary content
-    // Note : need to handle risk of index overflow
-    // Use only one memory segment for dict, so any previous External Dict is lost at this stage
-    ctxPtr->lowLimit  = ctxPtr->dictLimit;
-    ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
-    ctxPtr->dictBase  = ctxPtr->base;
-    ctxPtr->base = newBlock - ctxPtr->dictLimit;
-    ctxPtr->end  = newBlock;
-    ctxPtr->nextToUpdate = ctxPtr->dictLimit;   // reference table must skip to from beginning of block
-}
-
-
 FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4,   // Index table will be updated
                  const BYTE* ip, const BYTE* const iLimit,
                  const BYTE** matchpos,
@@ -618,8 +618,6 @@ static int LZ4HC_compress_generic (
     if (compressionLevel > MAX_COMPRESSION_LEVEL) compressionLevel = MAX_COMPRESSION_LEVEL;
     if (compressionLevel == 0) compressionLevel = LZ4HC_DEFAULT_COMPRESSIONLEVEL;
     maxNbAttempts = 1 << compressionLevel;
-    /* check if blocks follow each other */
-    if (ip != ctx->end) LZ4HC_setExternalDict(ctx, ip);
     ctx->end += inputSize;
 
     ip++;
@@ -866,16 +864,46 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
 
 /* compression */
 
+static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* dsPtr,
+                                            const char* source, char* dest,
+                                            int inputSize, int maxOutputSize, limitedOutput_directive limit)
+{
+    /* auto-init if forgotten */
+    if (dsPtr->base == NULL)
+        LZ4HC_init (dsPtr, (const BYTE*) source);
+
+    /* check if blocks follow each other */
+    if ((const BYTE*)source != dsPtr->end) LZ4HC_setExternalDict(dsPtr, (const BYTE*)source);
+
+	/* Check overlapping input/dictionary space */
+    {
+        const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+        const BYTE* dictBegin = dsPtr->dictBase + dsPtr->lowLimit;
+        const BYTE* dictEnd   = dsPtr->dictBase + dsPtr->dictLimit;
+        if ((sourceEnd > dictBegin) && (sourceEnd < dictEnd))
+        {
+            dsPtr->lowLimit = (U32)(sourceEnd - dsPtr->dictBase);
+            if (dsPtr->dictLimit - dsPtr->lowLimit < 4) dsPtr->lowLimit = dsPtr->dictLimit;
+        }
+    }
+
+    return LZ4HC_compress_generic (dsPtr, source, dest, inputSize, maxOutputSize, dsPtr->compressionLevel, limit);
+}
+
 int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize)
 {
-    if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
-    return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit);
+    //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL)
+    //    LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
+    //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, 0, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, noLimit);
+    return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit);
 }
 
 int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
 {
-    if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL) LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
-    return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput);
+    //if (((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base == NULL)
+    //    LZ4HC_init ((LZ4HC_Data_Structure*) LZ4_streamHCPtr, (const BYTE*) source);
+    //return LZ4HC_compress_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel, limitedOutput);
+    return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
 }