From 0c62103105143eaaf0fa5caae09e65318063a417 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 11 Jun 2014 21:40:16 +0100 Subject: restored LZ4 HC streaming mode --- lz4.c | 6 ++++-- lz4hc.c | 6 ++++-- programs/Makefile | 4 +++- programs/lz4io.c | 54 +++++++++++++++++++++++++++++++++++++++++++----------- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/lz4.c b/lz4.c index d2906aa..e1bcb25 100755 --- a/lz4.c +++ b/lz4.c @@ -47,8 +47,9 @@ **************************************/ /* 32 or 64 bits ? */ #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ - || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ + || defined(__powerpc64__) || defined(__powerpc64le__) \ + || defined(__ppc64__) || defined(__ppc64le__) \ + || defined(__PPC64__) || defined(__PPC64LE__) \ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */ # define LZ4_ARCH64 1 #else @@ -59,6 +60,7 @@ * Little Endian or Big Endian ? * Overwrite the #define below if you know your architecture endianess */ +#include /* Apparently required to detect endianess */ #if defined (__GLIBC__) # include # if (__BYTE_ORDER == __BIG_ENDIAN) diff --git a/lz4hc.c b/lz4hc.c index e84de2b..6086749 100644 --- a/lz4hc.c +++ b/lz4hc.c @@ -54,8 +54,9 @@ **************************************/ /* 32 or 64 bits ? */ #if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \ - || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \ + || defined(__powerpc64__) || defined(__powerpc64le__) \ + || defined(__ppc64__) || defined(__ppc64le__) \ + || defined(__PPC64__) || defined(__PPC64LE__) \ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */ # define LZ4_ARCH64 1 #else @@ -66,6 +67,7 @@ * Little Endian or Big Endian ? * Overwrite the #define below if you know your architecture endianess */ +#include /* Apparently required to detect endianess */ #if defined (__GLIBC__) # include # if (__BYTE_ORDER == __BIG_ENDIAN) diff --git a/programs/Makefile b/programs/Makefile index 9fb6cd9..35bfd06 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -158,8 +158,10 @@ test-fuzzer32: fuzzer32 ./fuzzer32 --no-prompt test-mem: lz4 datagen - ./datagen -g256M > tmp + ./datagen -g256MB > tmp valgrind ./lz4 -B4D -f tmp /dev/null + ./datagen -g16MB > tmp + valgrind ./lz4 -9 -B5D -f tmp /dev/null rm tmp test-mem32: lz4c32 datagen diff --git a/programs/lz4io.c b/programs/lz4io.c index a0dae68..650681b 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -365,12 +365,32 @@ int LZ4IO_compressFilename_Legacy(char* input_filename, char* output_filename, i } -static int compress_file_blockDependency2(char* input_filename, char* output_filename, int compressionlevel) +static void* LZ4IO_LZ4_createStream (const char* inputBuffer) { - void* (*initFunction) (); - int (*compressionFunction)(void*, const char*, char*, int, int); - int (*freeFunction) (void*); + (void)inputBuffer; + return LZ4_createStream(); +} + +static int LZ4IO_LZ4_compress_limitedOutput_continue (void* ctx, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +{ + (void)compressionLevel; + return LZ4_compress_limitedOutput_continue(ctx, source, dest, inputSize, maxOutputSize); +} + +static int LZ4IO_LZ4_slideInputBufferHC (void* ctx, char* buffer, int size) +{ + (void)size; (void)buffer; + LZ4_slideInputBufferHC (ctx); + return 1; +} + + +static int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) +{ + void* (*initFunction) (const char*); + int (*compressionFunction)(void*, const char*, char*, int, int, int); int (*nextBlockFunction) (void*, char*, int); + int (*freeFunction) (void*); void* ctx; unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; @@ -388,10 +408,20 @@ static int compress_file_blockDependency2(char* input_filename, char* output_fil start = clock(); if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; - initFunction = LZ4_createStream; - compressionFunction = LZ4_compress_limitedOutput_continue; - nextBlockFunction = LZ4_moveDict; - freeFunction = LZ4_free; + if (compressionlevel<3) + { + initFunction = LZ4IO_LZ4_createStream; + compressionFunction = LZ4IO_LZ4_compress_limitedOutput_continue; + nextBlockFunction = LZ4_moveDict; + freeFunction = LZ4_free; + } + else + { + initFunction = LZ4_createHC; + compressionFunction = LZ4_compressHC2_limitedOutput_continue; + nextBlockFunction = LZ4IO_LZ4_slideInputBufferHC; + freeFunction = LZ4_free; + } get_fileHandle(input_filename, output_filename, &finput, &foutput); blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); @@ -402,8 +432,9 @@ static int compress_file_blockDependency2(char* input_filename, char* output_fil out_buff = (char*)malloc(blockSize+CACHELINE); if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); in_blockStart = in_buff + 64 KB; + if (compressionlevel>=3) in_blockStart = in_buff; if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); - ctx = initFunction(); + ctx = initFunction(in_buff); // Write Archive Header *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention @@ -434,7 +465,7 @@ static int compress_file_blockDependency2(char* input_filename, char* output_fil if (streamChecksum) XXH32_update(streamChecksumState, in_blockStart, inSize); // Compress Block - outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1); + outSize = compressionFunction(ctx, in_blockStart, out_buff+4, inSize, inSize-1, compressionlevel); if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; if (blockChecksum) compressedfilesize+=4; DISPLAYLEVEL(3, "==> %.2f%% ", (double)compressedfilesize/filesize*100); @@ -472,6 +503,7 @@ static int compress_file_blockDependency2(char* input_filename, char* output_fil size_t sizeToMove = 64 KB; if (inSize < 64 KB) sizeToMove = inSize; nextBlockFunction(ctx, in_blockStart - sizeToMove, sizeToMove); + if (compressionlevel>=3) in_blockStart = in_buff + 64 KB; } } @@ -530,7 +562,7 @@ int LZ4IO_compressFilename(char* input_filename, char* output_filename, int comp void* streamChecksumState=NULL; // Branch out - if (blockIndependence==0) return compress_file_blockDependency2(input_filename, output_filename, compressionLevel); + if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionLevel); // Init start = clock(); -- cgit v0.12