diff options
| author | yann.collet.73@gmail.com <yann.collet.73@gmail.com@650e7d94-2a16-8b24-b05c-7c0b3f6821cd> | 2012-01-08 02:45:32 (GMT) |
|---|---|---|
| committer | yann.collet.73@gmail.com <yann.collet.73@gmail.com@650e7d94-2a16-8b24-b05c-7c0b3f6821cd> | 2012-01-08 02:45:32 (GMT) |
| commit | 97abf013482c8a8041e5938a80ceede566ef7d9d (patch) | |
| tree | 79c334fe7a00ba18a8be9372dc70f46276c591c9 | |
| parent | aac2572351dad4c48675091158620add2c0ff839 (diff) | |
| download | lz4-97abf013482c8a8041e5938a80ceede566ef7d9d.zip lz4-97abf013482c8a8041e5938a80ceede566ef7d9d.tar.gz lz4-97abf013482c8a8041e5938a80ceede566ef7d9d.tar.bz2 | |
Added : benchmark function within command-line utility
git-svn-id: https://lz4.googlecode.com/svn/trunk@46 650e7d94-2a16-8b24-b05c-7c0b3f6821cd
| -rw-r--r-- | Makefile | 8 | ||||
| -rw-r--r-- | bench.c | 352 | ||||
| -rw-r--r-- | bench.h | 36 | ||||
| -rw-r--r-- | lz4demo.c | 12 |
4 files changed, 402 insertions, 6 deletions
@@ -1,10 +1,10 @@ all: lz4demo64 lz4demo32 -lz4demo64: lz4.c lz4.h lz4demo.c - gcc -g -O3 -I. -std=c99 -Wall -W -Wno-implicit-function-declaration lz4.c lz4demo.c -o lz4demo64.exe +lz4demo64: lz4.c lz4.h bench.c lz4demo.c + gcc -g -O3 -I. -std=c99 -Wall -W -Wno-implicit-function-declaration lz4.c bench.c lz4demo.c -o lz4demo64.exe -lz4demo32: lz4.c lz4.h lz4demo.c - gcc -m32 -g -O3 -I. -std=c99 -Wall -W -Wno-implicit-function-declaration lz4.c lz4demo.c -o lz4demo32.exe +lz4demo32: lz4.c lz4.h bench.c lz4demo.c + gcc -m32 -g -O3 -I. -std=c99 -Wall -W -Wno-implicit-function-declaration lz4.c bench.c lz4demo.c -o lz4demo32.exe clean: rm -f core *.o lz4demo32.exe lz4demo64.exe @@ -0,0 +1,352 @@ +/*
+ bench.c - Demo program to benchmark open-source compression algorithm
+ Copyright (C) Yann Collet 2012
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+
+//**************************************
+// Compilation Directives
+//**************************************
+
+
+//**************************************
+// Includes
+//**************************************
+#include <stdio.h> // printf, fopen, fseeko64, ftello64
+#include <stdlib.h> // malloc
+#include <sys/timeb.h> // timeb
+#include "lz4.h"
+
+
+//**************************************
+// Basic Types
+//**************************************
+#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively
+#define BYTE unsigned __int8
+#define U16 unsigned __int16
+#define U32 unsigned __int32
+#define S32 __int32
+#define U64 unsigned __int64
+#else
+#include <stdint.h>
+#define BYTE uint8_t
+#define U16 uint16_t
+#define U32 uint32_t
+#define S32 int32_t
+#define U64 uint64_t
+#endif
+
+
+//**************************************
+// Constants
+//**************************************
+#define NBLOOPS 3
+#define TIMELOOP 2000
+
+#define KNUTH 2654435761U
+#define MAX_MEM (1984<<20)
+#define CHUNKSIZE (8<<20)
+#define MAX_NB_CHUNKS ((MAX_MEM / CHUNKSIZE) + 1)
+
+
+//**************************************
+// Local structures
+//**************************************
+struct chunkParameters
+{
+ U32 id;
+ char* inputBuffer;
+ char* outputBuffer;
+ int inputSize;
+ int outputSize;
+};
+
+struct compressionParameters
+{
+ int (*compressionFunction)(char*, char*, int);
+ int (*decompressionFunction)(char*, char*, int);
+};
+
+
+//**************************************
+// MACRO
+//**************************************
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+
+
+
+//*********************************************************
+// Private functions
+//*********************************************************
+
+
+static int BMK_GetMilliStart()
+{
+ // Supposed to be portable
+ // Rolls over every ~ 12.1 days (0x100000/24/60/60)
+ // Use GetMilliSpan to correct for rollover
+ struct timeb tb;
+ int nCount;
+ ftime( &tb );
+ nCount = tb.millitm + (tb.time & 0xfffff) * 1000;
+ return nCount;
+}
+
+
+static int BMK_GetMilliSpan( int nTimeStart )
+{
+ int nSpan = BMK_GetMilliStart() - nTimeStart;
+ if ( nSpan < 0 )
+ nSpan += 0x100000 * 1000;
+ return nSpan;
+}
+
+
+static U32 BMK_checksum(char* buff, U32 length)
+{
+ BYTE* p = (BYTE*)buff;
+ BYTE* bEnd = p + length;
+ BYTE* limit = bEnd - 3;
+ U32 idx = 1;
+ U32 crc = KNUTH;
+
+ while (p<limit)
+ {
+ crc += ((*(U32*)p) + idx++);
+ crc *= KNUTH;
+ p+=4;
+ }
+ while (p<bEnd)
+ {
+ crc += ((*p) + idx++);
+ crc *= KNUTH;
+ p++;
+ }
+ return crc;
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+ size_t step = (64U<<20); // 64 MB
+ BYTE* testmem=NULL;
+
+ requiredMem = (((requiredMem >> 25) + 1) << 26);
+ if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
+
+ requiredMem += 2*step;
+ while (!testmem)
+ {
+ requiredMem -= step;
+ testmem = malloc ((size_t)requiredMem);
+ }
+
+ free (testmem);
+ return (size_t) (requiredMem - step);
+}
+
+
+static U64 BMK_GetFileSize(FILE* f)
+{
+ U64 r;
+#ifdef _MSC_VER
+ _fseeki64(f, 0L, SEEK_END);
+ r = (U64) _ftelli64(f);
+ _fseeki64(f, 0L, SEEK_SET);
+#else
+ fseeko64(f, 0LL, SEEK_END);
+ r = (U64) ftello64(f);
+ fseeko64(f, 0LL, SEEK_SET);
+#endif
+ return r;
+}
+
+
+//*********************************************************
+// Public function
+//*********************************************************
+
+int BMK_benchFile(char** fileNamesTable, int nbFiles)
+{
+ int fileIdx=0;
+ FILE* fileIn;
+ char* infilename;
+ U64 largefilesize;
+ size_t benchedsize;
+ int nbChunks;
+ int maxCChunkSize;
+ size_t readSize;
+ char* in_buff;
+ char* out_buff; int out_buff_size;
+ struct chunkParameters chunkP[MAX_NB_CHUNKS];
+ U32 crcc, crcd;
+ struct compressionParameters compP;
+
+ U64 totals = 0;
+ U64 totalz = 0;
+ double totalc = 0.;
+ double totald = 0.;
+
+
+ // Init
+ compP.compressionFunction = LZ4_compress;
+ compP.decompressionFunction = LZ4_uncompress;
+
+ // Loop for each file
+ while (fileIdx<nbFiles)
+ {
+ // Check file existence
+ infilename = fileNamesTable[fileIdx++];
+ fileIn = fopen( infilename, "rb" );
+ if (fileIn==NULL)
+ {
+ DISPLAY( "Pb opening %s\n", infilename);
+ return 11;
+ }
+
+ // Memory allocation & restrictions
+ largefilesize = BMK_GetFileSize(fileIn);
+ benchedsize = (size_t) BMK_findMaxMem(largefilesize) / 2;
+ if ((U64)benchedsize > largefilesize) benchedsize = (size_t)largefilesize;
+ if (benchedsize < largefilesize)
+ {
+ DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", infilename, (int)(benchedsize>>20));
+ }
+
+ // Alloc
+ in_buff = malloc((size_t )benchedsize);
+ nbChunks = (benchedsize / CHUNKSIZE) + 1;
+ maxCChunkSize = CHUNKSIZE + CHUNKSIZE/255 + 64;
+ out_buff_size = nbChunks * maxCChunkSize;
+ out_buff = malloc((size_t )out_buff_size);
+
+ if(!in_buff || !out_buff)
+ {
+ DISPLAY("\nError: not enough memory!\n");
+ free(in_buff);
+ free(out_buff);
+ fclose(fileIn);
+ return 12;
+ }
+
+ // Init chunks data
+ {
+ int i;
+ size_t remaining = benchedsize;
+ char* in = in_buff;
+ char* out = out_buff;
+ for (i=0; i<nbChunks; i++)
+ {
+ chunkP[i].id = i;
+ chunkP[i].inputBuffer = in; in += CHUNKSIZE;
+ if (remaining > CHUNKSIZE) { chunkP[i].inputSize = CHUNKSIZE; remaining -= CHUNKSIZE; } else { chunkP[i].inputSize = remaining; remaining = 0; }
+ chunkP[i].outputBuffer = out; out += maxCChunkSize;
+ chunkP[i].outputSize = 0;
+ }
+ }
+
+ // Fill input buffer
+ DISPLAY("Loading %s... \r", infilename);
+ readSize = fread(in_buff, 1, benchedsize, fileIn);
+ fclose(fileIn);
+
+ if(readSize != benchedsize)
+ {
+ printf("\nError: problem reading file '%s' !! \n", infilename);
+ free(in_buff);
+ free(out_buff);
+ return 13;
+ }
+
+ // Calculating input Checksum
+ crcc = BMK_checksum(in_buff, benchedsize);
+
+
+ // Bench
+ {
+ int loopNb, nb_loops, chunkNb;
+ size_t cSize;
+ int milliTime;
+ double fastestC = 100000000., fastestD = 100000000.;
+
+ for (loopNb = 1; loopNb <= NBLOOPS; loopNb++)
+ {
+ // Compression
+ DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, infilename, (int)benchedsize);
+
+ nb_loops = 0;
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliStart() == milliTime);
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
+ {
+ for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
+ chunkP[chunkNb].outputSize = compP.compressionFunction(chunkP[chunkNb].inputBuffer, chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputSize);
+ nb_loops++;
+ }
+ milliTime = BMK_GetMilliSpan(milliTime);
+
+ if ((double)milliTime < fastestC*nb_loops) fastestC = (double)milliTime/nb_loops;
+ cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].outputSize;
+
+ DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%), %6.1f MB/s\r", loopNb, infilename, (int)benchedsize, (int)cSize, (double)cSize/(double)benchedsize*100., (double)benchedsize / fastestC / 1000.);
+
+ // Decompression
+ { size_t i; for (i=0; i<benchedsize; i++) in_buff[i]=0; } // zeroing area, for CRC checking
+
+ nb_loops = 0;
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliStart() == milliTime);
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
+ {
+ for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
+ chunkP[chunkNb].outputSize = compP.decompressionFunction(chunkP[chunkNb].outputBuffer, chunkP[chunkNb].inputBuffer, chunkP[chunkNb].inputSize);
+ nb_loops++;
+ }
+ milliTime = BMK_GetMilliSpan(milliTime);
+
+ if ((double)milliTime < fastestD*nb_loops) fastestD = (double)milliTime/nb_loops;
+ DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%), %6.1f MB/s , %6.1f MB/s\r", loopNb, infilename, (int)benchedsize, (int)cSize, (double)cSize/(double)benchedsize*100., (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.);
+ }
+
+ DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", infilename, (int)benchedsize, (int)cSize, (double)cSize/(double)benchedsize*100., (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.);
+ totals += benchedsize;
+ totalz += cSize;
+ totalc += fastestC;
+ totald += fastestD;
+
+ // CRC Checking
+ crcd = BMK_checksum(in_buff, benchedsize);
+ if (crcc!=crcd) printf("!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcc, (unsigned)crcd);
+ }
+
+ free(in_buff);
+ free(out_buff);
+ }
+
+ if (nbFiles > 1)
+ printf("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.);
+
+ return 0;
+}
+
+
+
@@ -0,0 +1,36 @@ +/*
+ bench.h - Demo program to benchmark open-source compression algorithm
+ Copyright (C) Yann Collet 2012
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
+ - LZ4 source repository : http://code.google.com/p/lz4/
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+int BMK_benchFile(char** fileNamesTable, int nbFiles) ;
+
+
+
+#if defined (__cplusplus)
+}
+#endif
@@ -39,6 +39,7 @@ #include <fcntl.h> // _O_BINARY
#endif
#include "lz4.h"
+#include "bench.h"
//**************************************
@@ -82,7 +83,8 @@ int usage() DISPLAY( "Arguments :\n");
DISPLAY( " -c : compression (default)\n");
DISPLAY( " -d : decompression \n");
- DISPLAY( " -t : test compressed file \n");
+ DISPLAY( " -b : benchmark with files\n");
+ DISPLAY( " -t : check compressed file \n");
DISPLAY( " -h : help (this text)\n");
DISPLAY( "input : can be 'stdin' (pipe) or a filename\n");
DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n");
@@ -271,7 +273,8 @@ int main(int argc, char** argv) {
int i,
compression=1, // default action if no argument
- decode=0;
+ decode=0,
+ bench=0;
char* input_filename=0;
char* output_filename=0;
#ifdef _WIN32
@@ -309,6 +312,9 @@ int main(int argc, char** argv) // Decoding
if ( argument[0] =='d' ) { decode=1; continue; }
+ // Bench
+ if ( argument[0] =='b' ) { bench=1; continue; }
+
// Test
if ( argument[0] =='t' ) { decode=1; output_filename=nulmark; continue; }
}
@@ -328,6 +334,8 @@ int main(int argc, char** argv) // No input filename ==> Error
if(!input_filename) { badusage(); return 1; }
+ if (bench) return BMK_benchFile(argv+2, argc-2);
+
// No output filename
if (!output_filename) { badusage(); return 1; }
|