From 647baabcef0effcfcb3cc0dadb2970db681c9d52 Mon Sep 17 00:00:00 2001 From: "yann.collet.73@gmail.com" Date: Sat, 30 Mar 2013 21:11:40 +0000 Subject: Updated : cmake/CMakeLists.txt, by Nobuhiro Iwamatsu Updated : cmake/pack/CMakeLists.txt, by Dmitry Cherepanov lz4demo : CLI accept aggregated commands lz4demo : detect overwrite output lz4demo : new commands options (-hc, -y) git-svn-id: https://lz4.googlecode.com/svn/trunk@91 650e7d94-2a16-8b24-b05c-7c0b3f6821cd --- cmake/CMakeLists.txt | 6 + cmake/pack/CMakeLists.txt | 2 +- cmake/release_COPYING.txt | 21 -- fuzzer.c | 115 ++++---- lz4.c | 1 + lz4.h | 2 +- lz4demo.c | 660 ++++++++++++++++++++++++++-------------------- 7 files changed, 437 insertions(+), 370 deletions(-) delete mode 100644 cmake/release_COPYING.txt diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 62bc347..3d3e558 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -83,6 +83,12 @@ if(BUILD_SHARED_LIBS) LIBRARY DESTINATION lib ARCHIVE DESTINATION lib ) + + install(FILES + ${SRC_DIR}/lz4.h + ${SRC_DIR}/lz4hc.h + DESTINATION include) + endif(BUILD_SHARED_LIBS) diff --git a/cmake/pack/CMakeLists.txt b/cmake/pack/CMakeLists.txt index 80adbb3..6d23235 100644 --- a/cmake/pack/CMakeLists.txt +++ b/cmake/pack/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required (VERSION 2.8) PROJECT(LZ4) ############################## CPACK -set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../) +set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 Packer") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 0) diff --git a/cmake/release_COPYING.txt b/cmake/release_COPYING.txt deleted file mode 100644 index 32cec0d..0000000 --- a/cmake/release_COPYING.txt +++ /dev/null @@ -1,21 +0,0 @@ - - lz4demo and fuzzer is an open-source demo compression algorithm LZ4 programs - Copyright (C) Yann Collet 2012 - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ diff --git a/fuzzer.c b/fuzzer.c index 11697fd..0a57f77 100644 --- a/fuzzer.c +++ b/fuzzer.c @@ -1,8 +1,8 @@ /* fuzzer.c - Fuzzer test tool for LZ4 Copyright (C) Andrew Mahone - Yann Collet 2012 - Original code by Andrew Mahone / Modified by Yann Collet - GPL v2 License + Original code by Andrew Mahone / Modified by Yann Collet + GPL v2 License This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,9 +18,9 @@ with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ */ //************************************** @@ -36,6 +36,7 @@ #include // fgets, sscanf #include // timeb #include "lz4.h" +#include "lz4hc.h" //************************************** @@ -79,8 +80,8 @@ static int FUZ_GetMilliSpan( int nTimeStart ) unsigned int FUZ_rand(unsigned int* src) { - *src = ((*src) * PRIME1) + PRIME2; - return *src; + *src = ((*src) * PRIME1) + PRIME2; + return *src; } @@ -110,7 +111,7 @@ int FUZ_SecurityTest() free(input); free(output); - printf(" Completed (r=%i)\n",r); + printf(" Completed (return = %i < 0)\n",r); return 0; } @@ -125,30 +126,30 @@ int main() { # define FUZ_avail ROUND_PAGE(FUZ_max) const int off_full = FUZ_avail - FUZ_max; unsigned char cbuf[FUZ_avail + PAGE_SIZE]; - unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); + unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); int i, j, k, ret, len; - char userInput[30] = {0}; + char userInput[30] = {0}; - printf("starting LZ4 fuzzer\n"); - printf("Select an Initialisation number (default : random) : "); - fflush(stdout); - if ( fgets(userInput, sizeof userInput, stdin) ) - { - if ( sscanf(userInput, "%d", &seed) == 1 ) {} - else seed = FUZ_GetMilliSpan(timestamp); - } - printf("Seed = %u\n", seed); + printf("starting LZ4 fuzzer\n"); + printf("Select an Initialisation number (default : random) : "); + fflush(stdout); + if ( fgets(userInput, sizeof userInput, stdin) ) + { + if ( sscanf(userInput, "%d", &seed) == 1 ) {} + else seed = FUZ_GetMilliSpan(timestamp); + } + printf("Seed = %u\n", seed); - FUZ_SecurityTest(); + FUZ_SecurityTest(); - for (i = 0; i < 2048; i++) + for (i = 0; i < 2048; i++) cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&seed) >> 16; for (i = 0; i < NB_ATTEMPTS; i++) { - printf("\r%7i /%7i\r", i, NB_ATTEMPTS); - - FUZ_rand(&seed); + printf("\r%7i /%7i\r", i, NB_ATTEMPTS); + + FUZ_rand(&seed); for (j = 0; j < NUM_SEQ; j++) { seeds[j] = FUZ_rand(&seed) << 8; seeds[j] ^= (FUZ_rand(&seed) >> 8) & 65535; @@ -165,63 +166,63 @@ int main() { buf[j] = FUZ_rand(&cur_seq) >> 16; } - // Test compression + // Test compression ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); - if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } len = ret; - // Test decoding with output size being exactly what's necessary => must work - ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN); - if (ret<0) { printf("decompression failed despite correct space: seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with output size being exactly what's necessary => must work + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN); + if (ret<0) { printf("decompression failed despite correct space: seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test decoding with one byte missing => must fail - ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1); - if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with one byte missing => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test decoding with one byte too much => must fail - ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1); - if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with one byte too much => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test decoding with enough output size => must work - ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); - if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with enough output size => must work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test decoding with output size being exactly what's necessary => must work - ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN); - if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with output size being exactly what's necessary => must work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test decoding with output size being one byte too short => must fail - ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); - if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with output size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test decoding with input size being one byte too short => must fail - ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); - if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with input size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test decoding with input size being one byte too large => must fail - ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); - if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + // Test decoding with input size being one byte too large => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test compression with output size being exactly what's necessary (should work) + // Test compression with output size being exactly what's necessary (should work) ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); goto _output_error; } if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } - // Test compression with just one missing byte into output buffer => must fail + // Test compression with just one missing byte into output buffer => must fail ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); goto _output_error; } if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); goto _output_error; } - bytes += LEN; + bytes += LEN; cbytes += len; } - printf("all tests completed successfully \n"); + printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); - getchar(); + getchar(); return 0; _output_error: - getchar(); - return 1; + getchar(); + return 1; } diff --git a/lz4.c b/lz4.c index 1f2eafd..f15dcfc 100644 --- a/lz4.c +++ b/lz4.c @@ -710,6 +710,7 @@ int LZ4_compress(const char* source, // are safe against "buffer overflow" attack type. // They will never write nor read outside of the provided output buffers. // LZ4_uncompress_unknownOutputSize() also insures that it will never read outside of the input buffer. +// LZ4_uncompress() guarantees that it will never read before source, nor beyond source + LZ4_compressBound(osize) // A corrupted input will produce an error result, a negative int, indicating the position of the error within input stream. int LZ4_uncompress(const char* source, diff --git a/lz4.h b/lz4.h index 4897eb2..b50e98d 100644 --- a/lz4.h +++ b/lz4.h @@ -66,7 +66,7 @@ LZ4_compress() : LZ4_uncompress() : osize : is the output size, therefore the original size - return : the number of bytes read in the source buffer + return : the number of bytes read in the source buffer (in other words, the compressed size) If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction This function never writes outside of provided buffers, and never modifies input buffer. note : destination buffer must be already allocated. diff --git a/lz4demo.c b/lz4demo.c index 85edcf0..9103c8b 100644 --- a/lz4demo.c +++ b/lz4demo.c @@ -1,31 +1,31 @@ /* - LZ4Demo - Demo CLI program using LZ4 compression - Copyright (C) Yann Collet 2011-2012 - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ + LZ4Demo - Demo CLI program using LZ4 compression + Copyright (C) Yann Collet 2011-2012 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ */ /* - Note : this is *only* a demo program, an example to show how LZ4 can be used. - It is not considered part of LZ4 compression library. - The license of LZ4 is BSD. - The license of the demo program is GPL. + Note : this is *only* a demo program, an example to show how LZ4 can be used. + It is not considered part of LZ4 compression library. + The license of LZ4 is BSD. + The license of the demo program is GPL. */ //************************************** @@ -63,11 +63,11 @@ #define swap32 __builtin_bswap32 #else static inline unsigned int swap32(unsigned int x) { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); - } + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} #endif @@ -102,301 +102,381 @@ static const int one = 1; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +//************************************** +// Special input/output +//************************************** +#define NULL_INPUT "null" +char stdinmark[] = "stdin"; +char stdoutmark[] = "stdout"; +#ifdef _WIN32 +char nulmark[] = "nul"; +#else +char nulmark[] = "/dev/null"; +#endif + + +//************************************** +// I/O Parameters +//************************************** +static int overwrite = 0; + + + //**************************** // Functions //**************************** int usage(char* exename) { - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] input output\n", exename); - DISPLAY( "Arguments :\n"); - DISPLAY( " -c0: Fast compression (default) \n"); - DISPLAY( " -c1: High compression \n"); - DISPLAY( " -d : decompression \n"); - DISPLAY( " -b#: Benchmark files, using # compression level\n"); - DISPLAY( " -t : check compressed file \n"); - DISPLAY( " -h : help (this text)\n"); - DISPLAY( "input : can be 'stdin' (pipe) or a filename\n"); - DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n"); - return 0; + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] input output\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " -c0/-c : Fast compression (default) \n"); + DISPLAY( " -c1/-hc: High compression \n"); + DISPLAY( " -d : decompression \n"); + DISPLAY( " -y : overwrite output \n"); + DISPLAY( " -t : check compressed file \n"); + DISPLAY( " -b# : Benchmark files, using # compression level\n"); + DISPLAY( " -H : help (this text)\n"); + DISPLAY( "input : can be 'stdin' (pipe) or a filename\n"); + DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n"); + return 0; } int badusage(char* exename) { - DISPLAY("Wrong parameters\n"); - usage(exename); - return 0; + DISPLAY("Wrong parameters\n"); + usage(exename); + return 0; } - int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) { - char stdinmark[] = "stdin"; - char stdoutmark[] = "stdout"; - if (!strcmp (input_filename, stdinmark)) { - DISPLAY( "Using stdin for input\n"); - *pfinput = stdin; + if (!strcmp (input_filename, stdinmark)) + { + DISPLAY( "Using stdin for input\n"); + *pfinput = stdin; #ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows - _setmode( _fileno( stdin ), _O_BINARY ); + _setmode( _fileno( stdin ), _O_BINARY ); #endif - } else { - *pfinput = fopen( input_filename, "rb" ); - } - - if (!strcmp (output_filename, stdoutmark)) { - DISPLAY( "Using stdout for output\n"); - *pfoutput = stdout; + } + else + { + *pfinput = fopen(input_filename, "rb"); + } + + if (!strcmp (output_filename, stdoutmark)) + { + DISPLAY( "Using stdout for output\n"); + *pfoutput = stdout; #ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows - _setmode( _fileno( stdout ), _O_BINARY ); + _setmode( _fileno( stdout ), _O_BINARY ); #endif - } else { - *pfoutput = fopen( output_filename, "wb" ); - } - - if ( *pfinput==0 ) { DISPLAY( "Pb opening %s\n", input_filename); return 2; } - if ( *pfoutput==0) { DISPLAY( "Pb opening %s\n", output_filename); return 3; } - - return 0; + } + else + { + // Check if destination file already exists + *pfoutput=0; + if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); + if (*pfoutput!=0) + { + char ch; + fclose(*pfoutput); + DISPLAY( "Warning : %s already exists\n", output_filename); + if (!overwrite) + { + DISPLAY( "Overwrite ? (Y/N) : "); + ch = getchar(); + if (ch!='Y') { DISPLAY( "Operation aborted : %s already exists\n", output_filename); return 12; } + } + } + *pfoutput = fopen( output_filename, "wb" ); + } + + if ( *pfinput==0 ) { DISPLAY( "Pb opening %s\n", input_filename); return 13; } + if ( *pfoutput==0) { DISPLAY( "Pb opening %s\n", output_filename); return 14; } + + return 0; } int compress_file(char* input_filename, char* output_filename, int compressionlevel) { - int (*compressionFunction)(const char*, char*, int); - unsigned long long filesize = 0; - unsigned long long compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE; - unsigned int u32var; - char* in_buff; - char* out_buff; - FILE* finput; - FILE* foutput; - int r; - int displayLevel = (compressionlevel>0); - clock_t start, end; - size_t sizeCheck; - - - // Init - switch (compressionlevel) - { - case 0 : compressionFunction = LZ4_compress; break; - case 1 : compressionFunction = LZ4_compressHC; break; - default : compressionFunction = LZ4_compress; - } - start = clock(); - r = get_fileHandle(input_filename, output_filename, &finput, &foutput); - if (r) return r; - - // Allocate Memory - in_buff = (char*)malloc(CHUNKSIZE); - out_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); - if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 8; } - - // Write Archive Header - u32var = ARCHIVE_MAGICNUMBER; - LITTLE_ENDIAN32(u32var); - *(unsigned int*)out_buff = u32var; - sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); - if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 10; } - - // Main Loop - while (1) - { - int outSize; - // Read Block - int inSize = (int) fread(in_buff, (size_t)1, (size_t)CHUNKSIZE, finput); - if( inSize<=0 ) break; - filesize += inSize; - if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20)); - - // Compress Block - outSize = compressionFunction(in_buff, out_buff+4, inSize); - compressedfilesize += outSize+4; - if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - // Write Block - LITTLE_ENDIAN32(outSize); - * (unsigned int*) out_buff = outSize; - LITTLE_ENDIAN32(outSize); - sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); - if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; } - } - - // Status - end = clock(); - DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close & Free - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; + int (*compressionFunction)(const char*, char*, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE; + unsigned int u32var; + char* in_buff; + char* out_buff; + FILE* finput; + FILE* foutput; + int r; + int displayLevel = (compressionlevel>0); + clock_t start, end; + size_t sizeCheck; + + + // Init + switch (compressionlevel) + { + case 0 : compressionFunction = LZ4_compress; break; + case 1 : compressionFunction = LZ4_compressHC; break; + default : compressionFunction = LZ4_compress; + } + start = clock(); + r = get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (r) return r; + + // Allocate Memory + in_buff = (char*)malloc(CHUNKSIZE); + out_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); + if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 21; } + + // Write Archive Header + u32var = ARCHIVE_MAGICNUMBER; + LITTLE_ENDIAN32(u32var); + *(unsigned int*)out_buff = u32var; + sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); + if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 22; } + + // Main Loop + while (1) + { + int outSize; + // Read Block + int inSize = (int) fread(in_buff, (size_t)1, (size_t)CHUNKSIZE, finput); + if( inSize<=0 ) break; + filesize += inSize; + if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20)); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, inSize); + compressedfilesize += outSize+4; + if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + LITTLE_ENDIAN32(outSize); + * (unsigned int*) out_buff = outSize; + LITTLE_ENDIAN32(outSize); + sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); + if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; } + } + + // Status + end = clock(); + DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; } int decode_file(char* input_filename, char* output_filename) { - unsigned long long filesize = 0; - char* in_buff; - char* out_buff; - size_t uselessRet; - int sinkint; - unsigned int chunkSize; - FILE* finput; - FILE* foutput; - clock_t start, end; - int r; - size_t sizeCheck; - - - // Init - start = clock(); - r = get_fileHandle(input_filename, output_filename, &finput, &foutput); - if (r) return r; - - // Allocate Memory - in_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); - out_buff = (char*)malloc(CHUNKSIZE); - if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 7; } - - // Check Archive Header - chunkSize = 0; - uselessRet = fread(&chunkSize, 1, ARCHIVE_MAGICNUMBER_SIZE, finput); - LITTLE_ENDIAN32(chunkSize); - if (chunkSize != ARCHIVE_MAGICNUMBER) { DISPLAY("Unrecognized header : file cannot be decoded\n"); return 6; } - - // Main Loop - while (1) - { - // Block Size - uselessRet = fread(&chunkSize, 1, 4, finput); - if( uselessRet==0 ) break; // Nothing to read : file read is completed - LITTLE_ENDIAN32(chunkSize); - if (chunkSize == ARCHIVE_MAGICNUMBER) - continue; // appended compressed stream - - // Read Block - uselessRet = fread(in_buff, 1, chunkSize, finput); - - // Decode Block - sinkint = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, chunkSize, CHUNKSIZE); - if (sinkint < 0) { DISPLAY("Decoding Failed ! Corrupted input !\n"); return 9; } - filesize += sinkint; - - // Write Block - sizeCheck = fwrite(out_buff, 1, sinkint, foutput); - if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 12; } - } - - // Status - end = clock(); - DISPLAY( "Successfully decoded %llu bytes \n", (unsigned long long)filesize); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close & Free - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; + unsigned long long filesize = 0; + char* in_buff; + char* out_buff; + size_t uselessRet; + int sinkint; + unsigned int chunkSize; + FILE* finput; + FILE* foutput; + clock_t start, end; + int r; + size_t sizeCheck; + + + // Init + start = clock(); + r = get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (r) return r; + + // Allocate Memory + in_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); + out_buff = (char*)malloc(CHUNKSIZE); + if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 31; } + + // Check Archive Header + chunkSize = 0; + uselessRet = fread(&chunkSize, 1, ARCHIVE_MAGICNUMBER_SIZE, finput); + LITTLE_ENDIAN32(chunkSize); + if (chunkSize != ARCHIVE_MAGICNUMBER) { DISPLAY("Unrecognized header : file cannot be decoded\n"); return 32; } + + // Main Loop + while (1) + { + // Block Size + uselessRet = fread(&chunkSize, 1, 4, finput); + if( uselessRet==0 ) break; // Nothing to read : file read is completed + LITTLE_ENDIAN32(chunkSize); + if (chunkSize == ARCHIVE_MAGICNUMBER) + continue; // appended compressed stream + + // Read Block + uselessRet = fread(in_buff, 1, chunkSize, finput); + + // Decode Block + sinkint = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, chunkSize, CHUNKSIZE); + if (sinkint < 0) { DISPLAY("Decoding Failed ! Corrupted input !\n"); return 33; } + filesize += sinkint; + + // Write Block + sizeCheck = fwrite(out_buff, 1, sinkint, foutput); + if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 34; } + } + + // Status + end = clock(); + DISPLAY( "Successfully decoded %llu bytes \n", (unsigned long long)filesize); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; } int main(int argc, char** argv) { - int i, - cLevel=0, - decode=0, - bench=0, - filenamesStart=2; - char* exename=argv[0]; - char* input_filename=0; - char* output_filename=0; -#ifdef _WIN32 - char nulmark[] = "nul"; -#else - char nulmark[] = "/dev/null"; -#endif - char nullinput[] = "null"; - - // Welcome message - DISPLAY( WELCOME_MESSAGE); - - if (argc<2) { badusage(exename); return 1; } - - for(i=1; i='0') cLevel=argument[1] - '0'; continue; } - - // Decoding - if ( argument[0] =='d' ) { decode=1; continue; } - - // Bench - if ( argument[0] =='b' ) { bench=1; if (argument[1] >= '0') cLevel=argument[1] - '0'; continue; } - - // Modify Block Size (benchmark only) - if ( argument[0] =='B' ) { int B = argument[1] - '0'; int S = 1 << (10 + 2*B); BMK_SetBlocksize(S); continue; } - - // Modify Nb Iterations (benchmark only) - if ( argument[0] =='i' ) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); continue; } - - // Pause at the end (benchmark only) - if ( argument[0] =='p' ) { BMK_SetPause(); continue; } - - // Test - if ( argument[0] =='t' ) { decode=1; output_filename=nulmark; continue; } - } - - // first provided filename is input - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - - // second provided filename is output - if (!output_filename) - { - output_filename=argument; - if (!strcmp (output_filename, nullinput)) output_filename = nulmark; - continue; - } - } - - // No input filename ==> Error - if(!input_filename) { badusage(exename); return 1; } - - if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); - - // No output filename ==> Error - if (!output_filename) { badusage(exename); return 1; } - - if (decode) return decode_file(input_filename, output_filename); - - return compress_file(input_filename, output_filename, cLevel); // Compression is 'default' action + int i, + cLevel=0, + decode=0, + bench=0, + filenamesStart=2; + char* exename=argv[0]; + char* input_filename=0; + char* output_filename=0; + char nullinput[] = NULL_INPUT; + char extension[] = EXTENSION; + + // Welcome message + DISPLAY( WELCOME_MESSAGE); + + if (argc<2) { badusage(exename); return 1; } + + for(i=1; i='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } break; + case 'h': if (argument[1]=='c') { cLevel=1; argument++; } break; + + // Decoding + case 'd': decode=1; break; + + // Bench + case 'b': bench=1; + if ((argument[1] >='0') && (argument[1] <='1')) { cLevel=argument[1] - '0'; argument++; } + break; + + // Modify Block Size (benchmark only) + case 'B': + if ((argument[1] >='0') && (argument[1] <='9')) + { + int B = argument[1] - '0'; + int S = 1 << (10 + 2*B); BMK_SetBlocksize(S); + argument++; + } + break; + + // Modify Nb Iterations (benchmark only) + case 'i': + if ((argument[1] >= '0') && (argument[1] <= '9')) + { + int iters = argument[1] - '0'; + BMK_SetNbIterations(iters); + argument++; + } + break; + + // Pause at the end (benchmark only) + case 'p': BMK_SetPause(); break; + + // Test + case 't': decode=1; output_filename=nulmark; break; + + // Overwrite + case 'y': overwrite=1; break; + + // Unrecognised command + default : badusage(exename); return 1; + } + } + continue; + } + + // first provided filename is input + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + + // second provided filename is output + if (!output_filename) + { + output_filename=argument; + if (!strcmp (output_filename, nullinput)) output_filename = nulmark; + continue; + } + } + + // No input filename ==> Error + if(!input_filename) { badusage(exename); return 1; } + + if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); + + // No output filename ==> build one automatically (for compression only) + if (!output_filename) + { + if (!decode) + { + int i=0, l=0; + while (input_filename[l]!=0) l++; + output_filename = (char*)calloc(1,l+5); + for (i=0;i