diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2015-03-23 07:48:42 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2015-03-23 07:48:42 (GMT) |
commit | 00c3208c7a352f4136300b257b418c55f07450c8 (patch) | |
tree | 51466bdcec0166a2e25d4dc276abac6f284d7f72 /lib | |
parent | e25b51de7b51101e04ceea194dd557fcc23c03ca (diff) | |
parent | 7f436a1215f11b0fb872c34f088b8b5888d0630d (diff) | |
download | lz4-00c3208c7a352f4136300b257b418c55f07450c8.zip lz4-00c3208c7a352f4136300b257b418c55f07450c8.tar.gz lz4-00c3208c7a352f4136300b257b418c55f07450c8.tar.bz2 |
Merge pull request #61 from Cyan4973/dev
Dev
Diffstat (limited to 'lib')
-rw-r--r-- | lib/Makefile | 10 | ||||
-rw-r--r-- | lib/README.md | 19 | ||||
-rw-r--r-- | lib/lz4.c | 52 | ||||
-rw-r--r-- | lib/lz4.h | 25 | ||||
-rw-r--r-- | lib/lz4frame.c | 518 | ||||
-rw-r--r-- | lib/lz4frame.h | 22 | ||||
-rw-r--r-- | lib/lz4frame_static.h | 6 | ||||
-rw-r--r-- | lib/lz4hc.c | 7 | ||||
-rw-r--r-- | lib/lz4hc.h | 6 | ||||
-rw-r--r-- | lib/xxhash.c | 3 | ||||
-rw-r--r-- | lib/xxhash.h | 3 |
11 files changed, 411 insertions, 260 deletions
diff --git a/lib/Makefile b/lib/Makefile index faffb39..8aa15de 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -26,20 +26,19 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # You can contact the author at : -# - LZ4 source repository : http://code.google.com/p/lz4/ -# - LZ4 source mirror : https://github.com/Cyan4973/lz4 +# - LZ4 source repository : https://github.com/Cyan4973/lz4 # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c # ################################################################ # Version numbers -VERSION ?= 126 +VERSION ?= 128 LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` LIBVER_PATCH=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) DESTDIR?= -PREFIX ?= /usr +PREFIX ?= /usr/local CFLAGS ?= -O3 CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic @@ -67,7 +66,7 @@ all: liblz4 liblz4: lz4.c lz4hc.c lz4frame.c xxhash.c @echo compiling static library - @$(CC) $(CPPFLAGS) $(CFLAGS) -c $^ + @$(CC) $(CPPFLAGS) $(CFLAGS) -fPIC -c $^ @$(AR) rcs liblz4.a lz4.o lz4hc.o lz4frame.o xxhash.o @echo compiling dynamic library $(LIBVER) @$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) @@ -112,6 +111,7 @@ uninstall: @[ -f $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4.h @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/lz4frame.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/lz4frame.h @echo lz4 libraries successfully uninstalled endif diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 0000000..636c8be --- /dev/null +++ b/lib/README.md @@ -0,0 +1,19 @@ +LZ4 - Library Files +================================ + +This directory contains many files, but you don't necessarily need them all. + +If you want to integrate LZ4 compression/decompression into your program, you basically need to include "**lz4.c**" and "**lz4.h**" only. + +If you want more compression, at the cost of compression speed (but preserving decompression speed), you will also have to include "**lz4hc.c**" and "**lz4hc.h**". Note that lz4hc needs lz4 to work properly. + +Next level, if you want to produce files or data streams compatible with lz4 utility, you will have to use and include "**lz4frame.c**" and **lz4frame.h**". This library encapsulate lz4-compressed blocks into the official interoperable frame format. In order to work properly, lz4frame needs lz4 and lz4hc, and also "**xxhash.c**" and "**xxhash.h**", which provide the error detection algorithm. + +A more complex "lz4frame_static.h" is also provided, although its usage is not recommended. It contains definitions which are not guaranteed to remain stable within future versions. Use only if you don't plan to update your lz4 version. + +The other files are not source code. There are : + + - LICENSE : contains the BSD license text + - Makefile : script to compile or install lz4 library (static or dynamic) + - liblz4.pc.in : for pkg-config (make install) + @@ -1,6 +1,7 @@ /* LZ4 - Fast LZ compression algorithm Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,8 +28,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4 - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -64,7 +64,7 @@ * Program will crash. * If uncommenting results in better performance (case 1) * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) - * An automatic detection macro will be added to match your case within future versions of the library. + * This way, an automatic detection macro can be added to match your case within later versions of the library. */ /* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ @@ -98,7 +98,7 @@ /************************************** - Compiler Options +* Compiler Options **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ /* "restrict" is a known keyword */ @@ -190,7 +190,7 @@ static U16 LZ4_readLE16(const void* memPtr) return *(U16*)memPtr; else { - const BYTE* p = memPtr; + const BYTE* p = (const BYTE*)memPtr; return (U16)((U16)p[0] + (p[1]<<8)); } } @@ -204,7 +204,7 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } else { - BYTE* p = memPtr; + BYTE* p = (BYTE*)memPtr; p[0] = (BYTE) value; p[1] = (BYTE)(value>>8); } @@ -285,9 +285,9 @@ static void LZ4_copy8(void* dstPtr, const void* srcPtr) /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { - BYTE* d = dstPtr; - const BYTE* s = srcPtr; - BYTE* e = dstEnd; + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* e = (BYTE*)dstEnd; do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e); } @@ -316,14 +316,14 @@ static const int LZ4_minLength = (MFLIMIT+1); /************************************** - Common Utils +* Common Utils **************************************/ #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -/******************************** - Common functions -********************************/ +/************************************** +* Common functions +**************************************/ static unsigned LZ4_NbCommonBytes (register size_t val) { if (LZ4_isLittleEndian()) @@ -412,7 +412,7 @@ static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLi #ifndef LZ4_COMMONDEFS_ONLY /************************************** - Local Constants +* Local Constants **************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) @@ -423,14 +423,14 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /************************************** - Local Utils +* Local Utils **************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } /************************************** - Local Structures and types +* Local Structures and types **************************************/ typedef struct { U32 hashTable[HASH_SIZE_U32]; @@ -453,10 +453,10 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; /******************************** - Compression functions +* Compression functions ********************************/ -static U32 LZ4_hashSequence(U32 sequence, tableType_t tableType) +static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -466,7 +466,7 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t tableType) static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); } -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { switch (tableType) { @@ -502,7 +502,7 @@ static int LZ4_compress_generic( int inputSize, int maxOutputSize, limitedOutput_directive outputLimited, - tableType_t tableType, + tableType_t const tableType, dict_directive dict, dictIssue_directive dictIssue) { @@ -749,7 +749,7 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in /***************************************** - Experimental : Streaming functions +* Experimental : Streaming functions *****************************************/ /* @@ -930,9 +930,9 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -/**************************** - Decompression functions -****************************/ +/******************************* +* Decompression functions +*******************************/ /* * This generic decompression function cover all use cases. * It shall be instantiated several times, using different sets of directives @@ -1151,7 +1151,7 @@ typedef struct */ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U64), LZ4_STREAMDECODESIZE_U64); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); return lz4s; } @@ -1282,7 +1282,7 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres /*************************************************** - Obsolete Functions +* Obsolete Functions ***************************************************/ /* These function names are deprecated and should no longer be used. @@ -1,7 +1,8 @@ /* LZ4 - Fast LZ compression algorithm Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once @@ -44,16 +45,16 @@ extern "C" { */ /************************************** - Version +* Version **************************************/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 5 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_MINOR 6 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); /************************************** - Tuning parameter +* Tuning parameter **************************************/ /* * LZ4_MEMORY_USAGE : @@ -66,7 +67,7 @@ int LZ4_versionNumber (void); /************************************** - Simple Functions +* Simple Functions **************************************/ int LZ4_compress (const char* source, char* dest, int sourceSize); @@ -95,7 +96,7 @@ LZ4_decompress_safe() : /************************************** - Advanced Functions +* Advanced Functions **************************************/ #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) @@ -169,7 +170,7 @@ int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedS /*********************************************** - Streaming Compression Functions +* Streaming Compression Functions ***********************************************/ #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) @@ -211,6 +212,7 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictS * LZ4_compress_continue * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio * Previous data blocks are assumed to still be present at their previous location. + * dest buffer must be already allocated, and sized to at least LZ4_compressBound(inputSize) */ int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); @@ -227,14 +229,13 @@ int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char * save it into a safer place (char* safeBuffer) * Note : you don't need to call LZ4_loadDict() afterwards, * dictionary is immediately usable, you can therefore call again LZ4_compress_continue() - * Return : dictionary size in bytes, or 0 if error - * Note : any dictSize > 64 KB will be interpreted as 64KB. + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error */ int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); /************************************************ - Streaming Decompression Functions +* Streaming Decompression Functions ************************************************/ #define LZ4_STREAMDECODESIZE_U64 4 @@ -285,7 +286,7 @@ int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalS /************************************** - Obsolete Functions +* Obsolete Functions **************************************/ /* Obsolete decompression functions diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 5183f22..644f3e8 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -1,6 +1,7 @@ /* LZ4 auto-framing library -Copyright (C) 2011-2014, Yann Collet. +Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,13 +28,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /* LZ4F is a stand-alone API to create LZ4-compressed Frames -* fully conformant to specification v1.4.1. -* All related operations, including memory management, are handled by the library. +* in full conformance with specification v1.5.0 +* All related operations, including memory management, are handled by the library. * */ @@ -44,15 +45,9 @@ Compiler Options # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#ifdef __GNUC__ -# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ -#endif - /************************************** -Memory routines +* Memory routines **************************************/ #include <stdlib.h> /* malloc, calloc, free */ #define ALLOCATOR(s) calloc(1,s) @@ -62,7 +57,7 @@ Memory routines /************************************** -Includes +* Includes **************************************/ #include "lz4frame_static.h" #include "lz4.h" @@ -71,7 +66,7 @@ Includes /************************************** -Basic Types +* Basic Types **************************************/ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include <stdint.h> @@ -90,7 +85,7 @@ typedef unsigned long long U64; /************************************** -Constants +* Constants **************************************/ #define KB *(1<<10) #define MB *(1<<20) @@ -102,36 +97,39 @@ Constants #define _4BITS 0x0F #define _8BITS 0xFF +#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U #define LZ4F_MAGICNUMBER 0x184D2204U #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LZ4F_MAXHEADERFRAME_SIZE 7 +#define LZ4F_MAXHEADERFRAME_SIZE 15 #define LZ4F_BLOCKSIZEID_DEFAULT max64KB +static const size_t minFHSize = 5; static const U32 minHClevel = 3; /************************************** -Structures and local types +* Structures and local types **************************************/ typedef struct { LZ4F_preferences_t prefs; - U32 version; - U32 cStage; + U32 version; + U32 cStage; size_t maxBlockSize; size_t maxBufferSize; BYTE* tmpBuff; BYTE* tmpIn; size_t tmpInSize; + U64 totalInSize; XXH32_state_t xxh; - void* lz4CtxPtr; - U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ + void* lz4CtxPtr; + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ } LZ4F_cctx_internal_t; typedef struct { LZ4F_frameInfo_t frameInfo; - unsigned version; - unsigned dStage; + U32 version; + U32 dStage; size_t maxBlockSize; size_t maxBufferSize; const BYTE* srcExpect; @@ -145,23 +143,18 @@ typedef struct size_t tmpOutSize; size_t tmpOutStart; XXH32_state_t xxh; - BYTE header[8]; + BYTE header[16]; } LZ4F_dctx_internal_t; /************************************** -Macros -**************************************/ - - -/************************************** -Error management +* Error management **************************************/ #define LZ4F_GENERATE_STRING(STRING) #STRING, static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; -U32 LZ4F_isError(LZ4F_errorCode_t code) +unsigned LZ4F_isError(LZ4F_errorCode_t code) { return (code > (LZ4F_errorCode_t)(-ERROR_maxCode)); } @@ -175,7 +168,7 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code) /************************************** -Private functions +* Private functions **************************************/ static size_t LZ4F_getBlockSize(unsigned blockSizeID) { @@ -189,6 +182,15 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID) /* unoptimized version; solves endianess & alignment issues */ +static U32 LZ4F_readLE32 (const BYTE* srcPtr) +{ + U32 value32 = srcPtr[0]; + value32 += (srcPtr[1]<<8); + value32 += (srcPtr[2]<<16); + value32 += (srcPtr[3]<<24); + return value32; +} + static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) { dstPtr[0] = (BYTE)value32; @@ -197,50 +199,70 @@ static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) dstPtr[3] = (BYTE)(value32 >> 24); } -static U32 LZ4F_readLE32 (const BYTE* srcPtr) +static U64 LZ4F_readLE64 (const BYTE* srcPtr) { - U32 value32 = srcPtr[0]; - value32 += (srcPtr[1]<<8); - value32 += (srcPtr[2]<<16); - value32 += (srcPtr[3]<<24); - return value32; + U64 value64 = srcPtr[0]; + value64 += (srcPtr[1]<<8); + value64 += (srcPtr[2]<<16); + value64 += (srcPtr[3]<<24); + value64 += ((U64)srcPtr[4]<<32); + value64 += ((U64)srcPtr[5]<<40); + value64 += ((U64)srcPtr[6]<<48); + value64 += ((U64)srcPtr[7]<<56); + return value64; +} + +static void LZ4F_writeLE64 (BYTE* dstPtr, U64 value64) +{ + dstPtr[0] = (BYTE)value64; + dstPtr[1] = (BYTE)(value64 >> 8); + dstPtr[2] = (BYTE)(value64 >> 16); + dstPtr[3] = (BYTE)(value64 >> 24); + dstPtr[4] = (BYTE)(value64 >> 32); + dstPtr[5] = (BYTE)(value64 >> 40); + dstPtr[6] = (BYTE)(value64 >> 48); + dstPtr[7] = (BYTE)(value64 >> 56); } -static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length) +static BYTE LZ4F_headerChecksum (const void* header, size_t length) { - U32 xxh = XXH32(header, (U32)length, 0); + U32 xxh = XXH32(header, length, 0); return (BYTE)(xxh >> 8); } /************************************** -Simple compression functions +* Simple compression functions **************************************/ +static blockSizeID_t LZ4F_optimalBSID(const blockSizeID_t requestedBSID, const size_t srcSize) +{ + blockSizeID_t proposedBSID = max64KB; + size_t maxBlockSize = 64 KB; + while (requestedBSID > proposedBSID) + { + if (srcSize <= maxBlockSize) + return proposedBSID; + proposedBSID = (blockSizeID_t)((int)proposedBSID + 1); + maxBlockSize <<= 2; + } + return requestedBSID; +} + + size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefs = { 0 }; + LZ4F_preferences_t prefs; size_t headerSize; size_t streamSize; if (preferencesPtr!=NULL) prefs = *preferencesPtr; - { - blockSizeID_t proposedBSID = max64KB; - size_t maxBlockSize = 64 KB; - while (prefs.frameInfo.blockSizeID > proposedBSID) - { - if (srcSize <= maxBlockSize) - { - prefs.frameInfo.blockSizeID = proposedBSID; - break; - } - proposedBSID++; - maxBlockSize <<= 2; - } - } + else memset(&prefs, 0, sizeof(prefs)); + + prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; - headerSize = 7; /* basic header size (no option) including magic number */ + headerSize = 15; /* header size, including magic number and frame content size*/ streamSize = LZ4F_compressBound(srcSize, &prefs); return headerSize + streamSize; @@ -248,43 +270,47 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere /* LZ4F_compressFrame() -* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.4.1, in a single step. +* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) -* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. +* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. * The result of the function is the number of bytes written into dstBuffer. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) */ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_cctx_internal_t cctxI = { 0 }; /* works because no allocation */ - LZ4F_preferences_t prefs = { 0 }; - LZ4F_compressOptions_t options = { 0 }; + LZ4F_cctx_internal_t cctxI; + LZ4_stream_t lz4ctx; + LZ4F_preferences_t prefs; + LZ4F_compressOptions_t options; LZ4F_errorCode_t errorCode; BYTE* const dstStart = (BYTE*) dstBuffer; BYTE* dstPtr = dstStart; BYTE* const dstEnd = dstStart + dstMaxSize; + memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ + memset(&options, 0, sizeof(options)); cctxI.version = LZ4F_VERSION; cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ if (preferencesPtr!=NULL) prefs = *preferencesPtr; + else { - blockSizeID_t proposedBSID = max64KB; - size_t maxBlockSize = 64 KB; - while (prefs.frameInfo.blockSizeID > proposedBSID) - { - if (srcSize <= maxBlockSize) - { - prefs.frameInfo.blockSizeID = proposedBSID; - break; - } - proposedBSID++; - maxBlockSize <<= 2; - } + memset(&prefs, 0, sizeof(prefs)); + prefs.frameInfo.frameOSize = (U64)srcSize; + } + if (prefs.frameInfo.frameOSize != 0) + prefs.frameInfo.frameOSize = (U64)srcSize; /* correct frame size if selected (!=0) */ + + if (prefs.compressionLevel < minHClevel) + { + cctxI.lz4CtxPtr = &lz4ctx; + cctxI.lz4CtxLevel = 1; } + + prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) prefs.frameInfo.blockMode = blockIndependent; /* no need for linked blocks */ @@ -298,8 +324,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; /* header size */ - dstMaxSize -= errorCode; - errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstMaxSize, srcBuffer, srcSize, &options); + errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; @@ -307,7 +332,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; - FREEMEM(cctxI.lz4CtxPtr); + if (prefs.compressionLevel >= minHClevel) /* no allocation necessary with lz4 fast */ + FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); } @@ -361,7 +387,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp */ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_preferences_t prefNull = { 0 }; + LZ4F_preferences_t prefNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -370,20 +396,21 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall; if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC; + memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; /* ctx Management */ { - U32 targetCtxLevel = cctxPtr->prefs.compressionLevel<minHClevel ? 1 : 2; - if (cctxPtr->lz4CtxLevel < targetCtxLevel) + U32 tableID = cctxPtr->prefs.compressionLevel<minHClevel ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + if (cctxPtr->lz4CtxLevel < tableID) { FREEMEM(cctxPtr->lz4CtxPtr); if (cctxPtr->prefs.compressionLevel<minHClevel) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); - cctxPtr->lz4CtxLevel = targetCtxLevel; + cctxPtr->lz4CtxLevel = tableID; } } @@ -405,7 +432,7 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; XXH32_reset(&(cctxPtr->xxh), 0); - if (cctxPtr->prefs.compressionLevel<minHClevel) + if (cctxPtr->prefs.compressionLevel < minHClevel) LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); else LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); @@ -418,13 +445,22 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds /* FLG Byte */ *dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */ + + (BYTE)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ + + (BYTE)((cctxPtr->prefs.frameInfo.frameOSize > 0) << 3); /* Frame content size */ /* BD Byte */ - *dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctxPtr->prefs.frameInfo.frameOSize) + { + LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.frameOSize); + dstPtr += 8; + cctxPtr->totalInSize = 0; + } /* CRC Byte */ - *dstPtr++ = LZ4F_headerChecksum(headerStart, 2); + *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); + dstPtr++; - cctxPtr->cStage = 1; /* header written, wait for data block */ + cctxPtr->cStage = 1; /* header written, now request input data block */ return (dstPtr - dstStart); } @@ -436,17 +472,19 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds * */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - const LZ4F_preferences_t prefsNull = { 0 }; - const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; - size_t blockSize = LZ4F_getBlockSize(bid); - unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; - size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; - size_t blockInfo = 4; /* default, without block CRC option */ - size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); - size_t result = (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd; - - return result; + LZ4F_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + { + const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LZ4F_getBlockSize(bid); + unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; + } } @@ -518,7 +556,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; */ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_compressOptions_t cOptionsNull = { 0 }; + LZ4F_compressOptions_t cOptionsNull; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; size_t blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; @@ -531,6 +569,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; /* select compression function */ @@ -611,8 +650,9 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d } if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) - XXH32_update(&(cctxPtr->xxh), srcBuffer, (unsigned)srcSize); + XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + cctxPtr->totalInSize += srcSize; return dstPtr - dstStart; } @@ -627,7 +667,6 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d */ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_compressOptions_t cOptionsNull = { 0 }; LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; @@ -637,7 +676,6 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; if (dstMaxSize < (cctxPtr->tmpInSize + 16)) return (size_t)-ERROR_dstMaxSize_tooSmall; - if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ @@ -691,13 +729,19 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + if (cctxPtr->prefs.frameInfo.frameOSize) + { + if (cctxPtr->prefs.frameInfo.frameOSize != cctxPtr->totalInSize) + return (size_t)-ERROR_frameSize_wrong; + } + return dstPtr - dstStart; } -/*********************************** -* Decompression functions -* *********************************/ +/********************************** +* Decompression functions +**********************************/ /* Resource management */ @@ -708,11 +752,11 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB * If the result LZ4F_errorCode_t is not zero, there was an error during context creation. * Object can release its memory using LZ4F_freeDecompressionContext(); */ -LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) { LZ4F_dctx_internal_t* dctxPtr; - dctxPtr = ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); + dctxPtr = (LZ4F_dctx_internal_t*)ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC; dctxPtr->version = versionNumber; @@ -720,7 +764,7 @@ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_compressionContext_t* LZ4F return OK_NoError; } -LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_decompressionContext) +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_decompressionContext) { LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)LZ4F_decompressionContext; FREEMEM(dctxPtr->tmpIn); @@ -730,51 +774,104 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_compressionContext_t LZ4F_de } -/* Decompression */ +/* ******************************************************************** */ +/* ********************* Decompression ******************************** */ +/* ******************************************************************** */ + +typedef enum { dstage_getHeader=0, dstage_storeHeader, + dstage_getCBlockSize, dstage_storeCBlockSize, + dstage_copyDirect, + dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, + dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + -static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPtr, size_t srcSize) +/* LZ4F_decodeHeader + return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) + or an error code (testable with LZ4F_isError()) + output : set internal values of dctx, such as + dctxPtr->frameInfo and dctxPtr->dStage. +*/ +static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) { BYTE FLG, BD, HC; - unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID; + unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; size_t bufferNeeded; + size_t frameHeaderSize; + const BYTE* srcPtr = (const BYTE*)srcVoidPtr; /* need to decode header to get frameInfo */ - if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */ + if (srcSize < minFHSize) return (size_t)-ERROR_GENERIC; /* minimal header size */ + memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); + + /* skippable frames */ + if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) + { + dctxPtr->frameInfo.frameType = skippableFrame; + if (srcVoidPtr == (void*)(dctxPtr->header)) + { + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + return srcSize; + } + else + { + dctxPtr->dStage = dstage_getSFrameSize; + return 4; + } + } /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_GENERIC; - srcPtr += 4; + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_frameType_unknown; + dctxPtr->frameInfo.frameType = LZ4F_frame; /* Flags */ - FLG = srcPtr[0]; - version = (FLG>>6)&_2BITS; + FLG = srcPtr[4]; + version = (FLG>>6) & _2BITS; blockMode = (FLG>>5) & _1BIT; blockChecksumFlag = (FLG>>4) & _1BIT; contentSizeFlag = (FLG>>3) & _1BIT; contentChecksumFlag = (FLG>>2) & _1BIT; - dictFlag = (FLG>>0) & _1BIT; - BD = srcPtr[1]; - blockSizeID = (BD>>4) & _3BITS; - /* check */ - HC = LZ4F_headerChecksum(srcPtr, 2); - if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ + /* Frame Header Size */ + frameHeaderSize = contentSizeFlag ? 15 : 7; + + if (srcSize < frameHeaderSize) + { + if (srcPtr != dctxPtr->header) + memcpy(dctxPtr->header, srcPtr, srcSize); + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = frameHeaderSize; + dctxPtr->dStage = dstage_storeHeader; + return srcSize; + } + + BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; /* validate */ - if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ + if (((FLG>>0)&_2BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* Only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ + if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ + + /* check */ + HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ /* save */ - dctxPtr->frameInfo.blockMode = blockMode; - dctxPtr->frameInfo.contentChecksumFlag = contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = blockSizeID; + dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID; dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + if (contentSizeFlag) + dctxPtr->frameInfo.frameOSize = LZ4F_readLE64(srcPtr+6); /* init */ if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); @@ -786,9 +883,9 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt FREEMEM(dctxPtr->tmpIn); FREEMEM(dctxPtr->tmpOutBuffer); dctxPtr->maxBufferSize = bufferNeeded; - dctxPtr->tmpIn = ALLOCATOR(dctxPtr->maxBlockSize); + dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC; - dctxPtr->tmpOutBuffer= ALLOCATOR(dctxPtr->maxBufferSize); + dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize); if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC; } dctxPtr->tmpInSize = 0; @@ -799,17 +896,10 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const BYTE* srcPt dctxPtr->tmpOutStart = 0; dctxPtr->tmpOutSize = 0; - return 7; -} - + dctxPtr->dStage = dstage_getCBlockSize; -typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_decodeHeader, - dstage_getCBlockSize, dstage_storeCBlockSize, dstage_decodeCBlockSize, - dstage_copyDirect, - dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, - dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, - dstage_getSuffix, dstage_storeSuffix, dstage_checkSuffix -} dStage_t; + return frameHeaderSize; +} /* LZ4F_getFrameInfo() @@ -830,11 +920,10 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont { LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcBuffer, *srcSizePtr); if (LZ4F_isError(errorCode)) return errorCode; - *srcSizePtr = errorCode; + *srcSizePtr = errorCode; /* nb Bytes consumed */ *frameInfoPtr = dctxPtr->frameInfo; dctxPtr->srcExpect = NULL; - dctxPtr->dStage = dstage_getCBlockSize; - return 4; + return 4; /* nextSrcSizeHint : 4 == block header size */ } /* frameInfo already decoded */ @@ -844,15 +933,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont } +/* redirector, with common prototype */ static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) { - (void)dictStart; - (void)dictSize; + (void)dictStart; (void)dictSize; return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); } - static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { if (dctxPtr->dictSize==0) @@ -880,15 +968,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ { -#if 0 - size_t savedDictSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; - memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart- savedDictSize, savedDictSize); - dctxPtr->dict = dctxPtr->tmpOutBuffer; - dctxPtr->dictSize = savedDictSize + dctxPtr->tmpOutStart + dstSize; - return; - -#else - size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; size_t copySize = 64 KB - dctxPtr->tmpOutSize; BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; @@ -900,7 +979,6 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s dctxPtr->dict = dctxPtr->tmpOutBuffer; dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; return; -#endif } if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ @@ -952,18 +1030,19 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, const LZ4F_decompressOptions_t* decompressOptionsPtr) { LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; - static const LZ4F_decompressOptions_t optionsNull = { 0 }; + LZ4F_decompressOptions_t optionsNull; const BYTE* const srcStart = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcStart + *srcSizePtr; const BYTE* srcPtr = srcStart; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* const dstEnd = dstStart + *dstSizePtr; BYTE* dstPtr = dstStart; - const BYTE* selectedIn=NULL; + const BYTE* selectedIn = NULL; unsigned doAnotherStage = 1; size_t nextSrcSizeHint = 1; + memset(&optionsNull, 0, sizeof(optionsNull)); if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; *srcSizePtr = 0; *dstSizePtr = 0; @@ -971,7 +1050,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, /* expect to continue decoding src buffer where it left previously */ if (dctxPtr->srcExpect != NULL) { - if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_GENERIC; + if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_wrongSrcPtr; } /* programmed as a state machine */ @@ -986,39 +1065,31 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, { if (srcEnd-srcPtr >= 7) { - selectedIn = srcPtr; - srcPtr += 7; - dctxPtr->dStage = dstage_decodeHeader; + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); + if (LZ4F_isError(errorCode)) return errorCode; + srcPtr += errorCode; break; } dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = 7; dctxPtr->dStage = dstage_storeHeader; - break; } case dstage_storeHeader: { - size_t sizeToCopy = 7 - dctxPtr->tmpInSize; + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); dctxPtr->tmpInSize += sizeToCopy; srcPtr += sizeToCopy; - if (dctxPtr->tmpInSize < 7) + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { - nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4; - doAnotherStage = 0; /* no enough src, wait to get some more */ + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; + doAnotherStage = 0; /* not enough src data, ask for some more */ break; } - selectedIn = dctxPtr->header; - dctxPtr->dStage = dstage_decodeHeader; - break; - } - - case dstage_decodeHeader: - { - LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, selectedIn, 7); + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); if (LZ4F_isError(errorCode)) return errorCode; - dctxPtr->dStage = dstage_getCBlockSize; break; } @@ -1028,15 +1099,16 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, { selectedIn = srcPtr; srcPtr += 4; - dctxPtr->dStage = dstage_decodeCBlockSize; - break; } + else + { /* not enough input to read cBlockSize field */ - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeCBlockSize; - break; + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlockSize; + } } + if (dctxPtr->dStage == dstage_storeCBlockSize) case dstage_storeCBlockSize: { size_t sizeToCopy = 4 - dctxPtr->tmpInSize; @@ -1051,11 +1123,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_decodeCBlockSize; - break; } - case dstage_decodeCBlockSize: + /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ { size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ @@ -1243,18 +1313,19 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, doAnotherStage = 0; break; } - if ((srcEnd - srcPtr) >= 4) /* CRC present */ + if ((srcEnd - srcPtr) < 4) /* not enough size for entire CRC */ + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeSuffix; + } + else { selectedIn = srcPtr; srcPtr += 4; - dctxPtr->dStage = dstage_checkSuffix; - break; } - dctxPtr->tmpInSize = 0; - dctxPtr->dStage = dstage_storeSuffix; - break; } + if (dctxPtr->dStage == dstage_storeSuffix) case dstage_storeSuffix: { size_t sizeToCopy = 4 - dctxPtr->tmpInSize; @@ -1269,11 +1340,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } selectedIn = dctxPtr->tmpIn; - dctxPtr->dStage = dstage_checkSuffix; - break; } - case dstage_checkSuffix: + /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ { U32 readCRC = LZ4F_readLE32(selectedIn); U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); @@ -1283,6 +1352,61 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, doAnotherStage = 0; break; } + + case dstage_getSFrameSize: + { + if ((srcEnd - srcPtr) >= 4) + { + selectedIn = srcPtr; + srcPtr += 4; + } + else + { + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 4; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + } + } + + if (dctxPtr->dStage == dstage_storeSFrameSize) + case dstage_storeSFrameSize: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* not enough input to get full sBlockSize; wait for more */ + { + nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctxPtr->header + 4; + } + + /* case dstage_decodeSBlockSize: */ /* no direct access */ + { + size_t SFrameSize = LZ4F_readLE32(selectedIn); + dctxPtr->frameInfo.frameOSize = SFrameSize; + dctxPtr->tmpInTarget = SFrameSize; + dctxPtr->dStage = dstage_skipSkippable; + break; + } + + case dstage_skipSkippable: + { + size_t skipSize = dctxPtr->tmpInTarget; + if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; + srcPtr += skipSize; + dctxPtr->tmpInTarget -= skipSize; + doAnotherStage = 0; + nextSrcSizeHint = dctxPtr->tmpInTarget; + if (nextSrcSizeHint) break; + dctxPtr->dStage = dstage_getHeader; + break; + } } } @@ -1320,10 +1444,12 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, } } - if (srcPtr<srcEnd) /* function must be called again with following source data */ + /* require function to be called again from position where it stopped */ + if (srcPtr<srcEnd) dctxPtr->srcExpect = srcPtr; else dctxPtr->srcExpect = NULL; + *srcSizePtr = (srcPtr - srcStart); *dstSizePtr = (dstPtr - dstStart); return nextSrcSizeHint; diff --git a/lib/lz4frame.h b/lib/lz4frame.h index d73e3e2..54db165 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -28,8 +28,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -66,19 +65,22 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code str typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7 } blockSizeID_t; typedef enum { blockLinked=0, blockIndependent} blockMode_t; typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t; +typedef enum { LZ4F_frame=0, skippableFrame } frameType_t; typedef struct { - blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - unsigned reserved[5]; + blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ + unsigned long long frameOSize; /* Size of uncompressed (original) content ; 0 == unknown */ + unsigned reserved[2]; /* must be zero for forward compatibility */ } LZ4F_frameInfo_t; typedef struct { LZ4F_frameInfo_t frameInfo; unsigned compressionLevel; /* 0 == default (fast mode); values above 16 count as 16 */ - unsigned autoFlush; /* 1 == always flush : reduce need for tmp buffer */ - unsigned reserved[4]; + unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ + unsigned reserved[4]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; @@ -130,7 +132,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr); /* LZ4F_compressBegin() : * will write the frame header into dstBuffer. - * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 19 bytes. + * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes. * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. * The result of the function is the number of bytes written into dstBuffer for the header * or an error code (can be tested using LZ4F_isError()) @@ -215,7 +217,7 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t ctx, * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError(). + * or an error code which can be tested using LZ4F_isError(). */ size_t LZ4F_decompress(LZ4F_decompressionContext_t ctx, diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index cde8186..2e56400 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -29,8 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 source mirror : https://github.com/Cyan4973/lz4 + - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -53,6 +52,9 @@ extern "C" { ITEM(ERROR_compressionLevel_invalid) \ ITEM(ERROR_allocation_failed) \ ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_frameType_unknown) \ + ITEM(ERROR_wrongSrcPtr) \ ITEM(ERROR_decompressionFailed) \ ITEM(ERROR_checksum_invalid) \ ITEM(ERROR_maxCode) diff --git a/lib/lz4hc.c b/lib/lz4hc.c index 5549969..357fa96 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1,6 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 -Copyright (C) 2011-2014, Yann Collet. +Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,8 +28,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html -- LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/lz4hc.h b/lib/lz4hc.h index ce813ab..eb72051 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -1,7 +1,7 @@ /* LZ4 HC - High Compression Mode of LZ4 Header File - Copyright (C) 2011-2014, Yann Collet. + Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,8 +28,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once diff --git a/lib/xxhash.c b/lib/xxhash.c index 093564c..aca1e0a 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -28,8 +28,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- xxHash source mirror : https://github.com/Cyan4973/xxHash +- xxHash source repository : https://github.com/Cyan4973/xxHash - public discussion board : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/lib/xxhash.h b/lib/xxhash.h index 55b4501..99b0c27 100644 --- a/lib/xxhash.h +++ b/lib/xxhash.h @@ -1,7 +1,8 @@ /* xxHash - Extremely Fast Hash algorithm Header File - Copyright (C) 2012-2014, Yann Collet. + Copyright (C) 2012-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without |