Added : function LZ4_compressHC_limitedOutput()

Updated : LZ4 Streaming Format.odt, to version 1.4
New : LZ4c now supports Stream Checksum (default) and Skippable chunks
Updated : Fuzzer, testing LZ4_compressHC_limitedOutput()

git-svn-id: https://lz4.googlecode.com/svn/trunk@93 650e7d94-2a16-8b24-b05c-7c0b3f6821cd

1 files changed, 76 insertions, 27 deletions

diff --git a/lz4c.c b/lz4c.c
index ebbb357..85c074a 100644
--- a/lz4c.c
+++ b/lz4c.c
@@ -92,6 +92,9 @@
 #define LZ4S_BLOCKSIZEID_DEFAULT 7

 #define LZ4S_CHECKSUM_SEED 0

 

+#define LZ4S_SKIPPABLE0    0x184D2A50

+#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0

+

 #define LEGACY_MAGICNUMBER 0x184C2102

 #define LEGACY_BLOCKSIZE (8<<20)          // 8 MB

 

@@ -134,8 +137,8 @@ char nulmark[] = "/dev/null";
 //**************************************

 static int overwrite = 0;

 static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT;

-static int blockChecksum = 1;

-

+static int blockChecksum = 0;

+static int streamChecksum = 1;

 

 //**************************************

 // Exceptions

@@ -163,19 +166,26 @@ int usage(char* exename)
     DISPLAY( "Arguments :\n");

     DISPLAY( " -c0/-c : Fast compression (default) \n");

     DISPLAY( " -c1/-hc: High compression \n");

-    DISPLAY( " -B#    : Block size [4-7](default:7)\n");

-    DISPLAY( " -x     : disable checksum\n");

-    DISPLAY( " -y     : overwrite without prompting \n");

     DISPLAY( " -d     : decompression \n");

-    DISPLAY( " -t     : check compressed file \n");

-    DISPLAY( " -b#    : benchmark files, using # [0-1] compression level\n");

-    DISPLAY( " -i#    : nb of loops [1-9](default : 3), benchmark mode only\n");

-    DISPLAY( " -H     : Help (this text)\n");

+    DISPLAY( " -y     : overwrite without prompting \n");

+    DISPLAY( " -H     : Help (this text + advanced options)\n");

     DISPLAY( "input   : can be 'stdin' (pipe) or a filename\n");

     DISPLAY( "output  : can be 'stdout'(pipe) or a filename or 'null'\n");

     return 0;

 }

 

+int usage_advanced()

+{

+    DISPLAY( "\nAdvanced options :\n");

+    DISPLAY( " -t     : test compressed file \n");

+    DISPLAY( " -B#    : Block size [4-7](default : 7)\n");

+    DISPLAY( " -x     : enable block checksum (default:disabled)\n");

+    DISPLAY( " -nx    : disable stream checksum (default:enabled)\n");

+    DISPLAY( " -b#    : benchmark files, using # [0-1] compression level\n");

+    DISPLAY( " -i#    : iteration loops [1-9](default : 3), benchmark mode only\n");

+    return 0;

+}

+

 

 int badusage(char* exename)

 {

@@ -317,7 +327,7 @@ int legacy_compress_file(char* input_filename, char* output_filename, int compre
 

 int compress_file(char* input_filename, char* output_filename, int compressionlevel)

 {

-    int (*compressionFunction)(const char*, char*, int);

+    int (*compressionFunction)(const char*, char*, int, int);

     unsigned long long filesize = 0;

     unsigned long long compressedfilesize = 0;

     unsigned int checkbits;

@@ -330,15 +340,16 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
     clock_t start, end;

     int blockSize;

     size_t sizeCheck, header_size;

+    void* streamChecksumState=NULL;

 

 

     // Init

     start = clock();

     switch (compressionlevel)

     {

-    case 0 : compressionFunction = LZ4_compress; break;

-    case 1 : compressionFunction = LZ4_compressHC; break;

-    default : compressionFunction = LZ4_compress;

+    case 0 : compressionFunction = LZ4_compress_limitedOutput; break;

+    case 1 : compressionFunction = LZ4_compressHC_limitedOutput; break;

+    default : compressionFunction = LZ4_compress_limitedOutput;

     }

     errorcode = get_fileHandle(input_filename, output_filename, &finput, &foutput);

     if (errorcode) return errorcode;

@@ -346,13 +357,15 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
 

     // Allocate Memory

     in_buff  = (char*)malloc(blockSize);

-    out_buff = (char*)malloc(LZ4_compressBound(blockSize));

+    out_buff = (char*)malloc(blockSize+CACHELINE);

     if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory");

+    if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);

 

     // Write Archive Header

      *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER);   // Magic Number, in Little Endian convention    

     *(out_buff+4)  = 0x60;                    // Version("01"), Block independence

     *(out_buff+4) |= blockChecksum << 4;     

+    *(out_buff+4) |= streamChecksum << 2;     

     *(out_buff+5)  = (char)(blockSizeId<<4);  // Block Size

     checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED);

     checkbits = LZ4S_GetCheckBits_FromXXH(checkbits);

@@ -371,15 +384,16 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
         if( inSize<=0 ) break;

         filesize += inSize;

         if (displayLevel) DISPLAY("Read : %i MB  \r", (int)(filesize>>20));

+        if (streamChecksum) XXH32_update(streamChecksumState, in_buff, inSize);

 

         // Compress Block

-        outSize = compressionFunction(in_buff, out_buff+4, inSize);

-        if (outSize < inSize) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;

+        outSize = compressionFunction(in_buff, out_buff+4, inSize, inSize-1);

+        if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4;

         if (blockChecksum) compressedfilesize+=4;

         if (displayLevel) DISPLAY("Read : %i MB  ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100);

 

         // Write Block

-        if (outSize < inSize)

+        if (outSize > 0)

         {

             unsigned int checksum;

             int sizeToWrite;

@@ -417,6 +431,14 @@ int compress_file(char* input_filename, char* output_filename, int compressionle
     sizeCheck = fwrite(out_buff, 1, 4, foutput);

     if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream");

     compressedfilesize += 4;

+    if (streamChecksum)

+    {

+        unsigned int checksum = XXH32_digest(streamChecksumState);

+        * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum);

+        sizeCheck = fwrite(out_buff, 1, 4, foutput);

+        if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum");

+        compressedfilesize += 4;

+    }

 

     // Status

     end = clock();

@@ -497,7 +519,8 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
     int decodedBytes;

     unsigned int maxBlockSize;

     size_t sizeCheck;

-    int blockChecksumFlag;

+    int blockChecksumFlag, streamChecksumFlag;

+    void* streamChecksumState=NULL;

 

     // Decode stream descriptor

     nbReadBytes = fread(descriptor, 1, 3, finput);

@@ -506,7 +529,7 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
         int version       = (descriptor[0] >> 6) & _2BITS;

         int independance  = (descriptor[0] >> 5) & _1BIT;

         int streamSize    = (descriptor[0] >> 3) & _1BIT;

-        int reserved1     = (descriptor[0] >> 1) & _2BITS;

+        int reserved1     = (descriptor[0] >> 1) & _1BIT;

         int dictionary    = (descriptor[0] >> 0) & _1BIT;

 

         int reserved2     = (descriptor[1] >> 7) & _1BIT;

@@ -516,6 +539,7 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
         int checkBits_xxh32;

 

         blockChecksumFlag = (descriptor[0] >> 4) & _1BIT;

+        streamChecksumFlag= (descriptor[0] >> 2) & _1BIT;

 

         if (version != 1)       EXM_THROW(62, "Wrong version number");

         if (independance != 1)  EXM_THROW(63, "Does not support block inter-dependence");

@@ -537,6 +561,7 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
     in_buff = (char*)malloc(maxBlockSize);

     out_buff = (char*)malloc(maxBlockSize);

     if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory");

+    if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED);

 

     // Main Loop

     while (1)

@@ -561,10 +586,10 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
         {

             unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED);

             unsigned int readChecksum;

-            nbReadBytes = fread(&readChecksum, 1, 4, finput);

-            if( nbReadBytes != 4 ) EXM_THROW(74, "Read error : cannot read next block size");

+            sizeCheck = fread(&readChecksum, 1, 4, finput);

+            if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size");

             readChecksum = LITTLE_ENDIAN_32(readChecksum);   // Convert to little endian

-            if (checksum != readChecksum) EXM_THROW(75, "Error : invalid checksum detected");

+            if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected");

         }

 

         if (uncompressedFlag)

@@ -573,6 +598,7 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
             sizeCheck = fwrite(in_buff, 1, blockSize, foutput);

             if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block");

             filesize += blockSize;

+            if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize);

         }

         else

         {

@@ -580,6 +606,7 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
             decodedBytes = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, blockSize, maxBlockSize);

             if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !");

             filesize += decodedBytes;

+            if (streamChecksumFlag) XXH32_update(streamChecksumState, out_buff, decodedBytes);

 

             // Write Block

             sizeCheck = fwrite(out_buff, 1, decodedBytes, foutput);

@@ -587,6 +614,17 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
         }

     }

 

+    // Stream Checksum

+    if (streamChecksumFlag)

+    {

+        unsigned int checksum = XXH32_digest(streamChecksumState);

+        unsigned int readChecksum;

+        sizeCheck = fread(&readChecksum, 1, 4, finput);

+        if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum");

+        readChecksum = LITTLE_ENDIAN_32(readChecksum);   // Convert to little endian

+        if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected");

+    }

+

     // Free

     free(in_buff);

     free(out_buff);

@@ -597,7 +635,8 @@ unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
 

 unsigned long long selectDecoder( FILE* finput,  FILE* foutput)

 {

-    unsigned int magicNumber;

+    unsigned int magicNumber, size;

+    int errorNb;

     size_t nbReadBytes;

 

     // Check Archive Header

@@ -605,6 +644,8 @@ unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
     if (nbReadBytes==0) return 0;                    // EOF

     if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable");

     magicNumber = LITTLE_ENDIAN_32(magicNumber);     // Convert to Little Endian format

+    if ((magicNumber & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0) magicNumber = LZ4S_SKIPPABLE0;  // fold skippable magic numbers

+

     switch(magicNumber)

     {

     case LZ4S_MAGICNUMBER:

@@ -612,8 +653,15 @@ unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
     case LEGACY_MAGICNUMBER:

         DISPLAY("Detected : Legacy format \n");

         return decodeLegacyStream(finput, foutput);

+    case LZ4S_SKIPPABLE0:

+        nbReadBytes = fread(&size, 1, 4, finput);

+        if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");

+        size = LITTLE_ENDIAN_32(size);     // Convert to Little Endian format

+        errorNb = fseek(finput, size, SEEK_CUR);

+        if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");

+        return selectDecoder(finput, foutput);

     default:

-        if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(42,"Unrecognized header : file cannot be decoded");

+        if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded");  // Wrong magic number at the beginning of 1st stream

         DISPLAY("Stream followed by unrecognized data\n");

         return 0;        

     }

@@ -691,14 +739,15 @@ int main(int argc, char** argv)
                 switch(argument[0])

                 {

                     // Display help on usage

-                case 'H': usage(exename); return 0;

+                case 'H': usage(exename); usage_advanced(); return 0;

 

                     // Compression (default)

                 case 'c': if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } break;

                 case 'h': if (argument[1]=='c') { cLevel=1; argument++; } break;

 

-                    // disable checksum

-                case 'x': blockChecksum=0; break;

+                    // Checksum management

+                case 'x': blockChecksum=1; break;

+                case 'n': if (argument[1]=='x') { streamChecksum=0; argument++; break; } else { badusage(exename); return 1; }

 

                     // Use Legacy format (hidden option)

                 case 'l': legacy_format=1; break;