diff options
author | Yann Collet <yann.collet.73@gmail.com> | 2015-05-07 11:23:39 (GMT) |
---|---|---|
committer | Yann Collet <yann.collet.73@gmail.com> | 2015-05-07 11:23:39 (GMT) |
commit | 7d182b816ace89d6d6d16b7aee376a9962a05caa (patch) | |
tree | 5b2236b59cd0e42e3365b0765041a25c85519b39 | |
parent | 160661c7a4cbf805f4af74d2e3932a17a66e6ce7 (diff) | |
parent | fdd0029c3778768d0e8a80ce238d981f946c8b78 (diff) | |
download | lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.zip lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.tar.gz lz4-7d182b816ace89d6d6d16b7aee376a9962a05caa.tar.bz2 |
Dev
49 files changed, 3393 insertions, 1674 deletions
diff --git a/.travis.yml b/.travis.yml index 8fabb1c..398a0f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,11 @@ before_install: - sudo apt-get install -qq valgrind env: - - LZ4_TRAVIS_CI_ENV=dist - LZ4_TRAVIS_CI_ENV=travis-install - LZ4_TRAVIS_CI_ENV=streaming-examples - LZ4_TRAVIS_CI_ENV=cmake - LZ4_TRAVIS_CI_ENV=clangtest + - LZ4_TRAVIS_CI_ENV=sanitize - LZ4_TRAVIS_CI_ENV=staticAnalyze - LZ4_TRAVIS_CI_ENV=gpptest - LZ4_TRAVIS_CI_ENV=armtest diff --git a/LZ4_Frame_Format.html b/LZ4_Frame_Format.html deleted file mode 100644 index 6622dbb..0000000 --- a/LZ4_Frame_Format.html +++ /dev/null @@ -1 +0,0 @@ -<html><head><title>LZ4 Framing format - v1.5.0</title><meta content="text/html; charset=UTF-8" http-equiv="content-type"><style type="text/css">@import url('https://themes.googleusercontent.com/fonts/css?kit=wAPX1HepqA24RkYW1AuHYA');ol{margin:0;padding:0}.c2{font-size:10pt;font-family:"Courier New";font-weight:bold}.c10{max-width:453.6pt;background-color:#ffffff;padding:70.8pt 70.8pt 70.8pt 70.8pt}.c4{line-height:1.0;height:11pt;padding-bottom:0pt}.c0{direction:ltr;margin-left:18pt}.c13{line-height:1.0;padding-bottom:0pt}.c8{color:inherit;text-decoration:inherit}.c3{color:#1155cc;text-decoration:underline}.c6{text-decoration:underline;font-weight:bold}.c14{font-weight:bold}.c18{font-family:"Courier New"}.c16{font-size:18pt}.c12{margin-left:36pt}.c11{font-size:14pt}.c5{height:11pt}.c9{text-align:center}.c7{text-decoration:underline}.c17{color:#0000ff}.c1{direction:ltr}.c15{font-style:italic}.title{padding-top:12pt;line-height:1.15;text-align:center;color:#000000;font-size:16pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}.subtitle{padding-top:0pt;line-height:1.15;text-align:center;color:#000000;font-size:11pt;font-family:"Arial";padding-bottom:3pt}li{color:#000000;font-size:11pt;font-family:"Calibri"}p{color:#000000;font-size:11pt;margin:0;font-family:"Calibri"}h1{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:16pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}h2{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-style:italic;font-size:14pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}h3{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:13pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}h4{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:14pt;font-family:"Calibri";font-weight:bold;padding-bottom:3pt}h5{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-style:italic;font-size:13pt;font-family:"Calibri";font-weight:bold;padding-bottom:3pt}h6{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:11pt;font-family:"Calibri";font-weight:bold;padding-bottom:3pt}</style></head><body class="c10"><hr><p class="c1 c9"><span class="c14 c16">LZ4 </span><span class="c14 c16">Framing </span><span class="c14 c16">Format</span></p><hr><p class="c5 c1"><span class="c14 c16"></span></p><p class="c1"><span class="c6 c11">Notices</span></p><p class="c1"><span>Copyright (c) 2013-2015 Yann Collet</span></p><p class="c1"><span>Permission is granted to copy and distribute this document for any purpose and without charge, including translations into other languages and incorporation into compilations, provided that the copyright notice and this notice are preserved, and that any substantive changes or deletions from the original are clearly marked.</span></p><p class="c1"><span class="c6 c11">Version</span></p><p class="c1"><span>1.5.0</span></p><h1 class="c1"><a name="h.2z5bl598dfq9"></a><span>Introduction</span></h1><p class="c1"><span>The purpose of this document is to define a lossless compressed data format, that is independent of CPU type, operating system, file system and character set, suitable for File compression, Pipe and streaming compression using the LZ4 algorithm : </span><span class="c7 c17"><a class="c8" href="">http://code.google.com/p/lz4/</a></span></p><p class="c1"><span>The data can be produced or consumed, even for an arbitrarily long sequentially presented input data stream, using only an a priori bounded amount of intermediate storage, and hence can be used in data communications. The format uses the LZ4 compression method, and optional </span><span class="c3"><a class="c8" href="http://code.google.com/p/xxhash/">xxHash-32</a></span><span> checksum method, for detection of data corruption.</span></p><p class="c1"><span>The data format defined by this specification does not attempt to allow random access to compressed data.</span></p><p class="c1"><span>This specification is intended for use by implementers of software to compress data into LZ4 format and/or decompress data from LZ4 format. The text of the specification assumes a basic background in programming at the level of bits and other primitive data representations.</span></p><p class="c1"><span>Unless otherwise indicated below, </span><span>a compliant compressor must produce data sets that conform to the specifications presented here. It doesn’t need to support all options though.</span></p><p class="c1"><span>A</span><span> compliant decompressor must be able to decompress </span><span>at least one working </span><span>set of parameters that conforms to the specifications presented here</span><span>. It may also ignore checksums. Whenever it does not support a specific parameter used within the compressed stream, it must produce a non-ambiguous error code and associated error message explaining which parameter is unsupported.</span></p><p class="c1"><span>Distribution of this document is unlimited.</span></p><p class="c4 c1"><span></span></p><hr style="page-break-before:always;display:none;"><p class="c1 c4"><span></span></p><p class="c1 c13"><span class="c6 c11">Summary </span><span class="c6">:</span></p><p class="c4 c1"><span></span></p><p class="c0"><span class="c3"><a class="c8" href="#h.2z5bl598dfq9">Introduction</a></span></p><p class="c0"><span class="c3">General structure of </span><span class="c3"><a class="c8" href="#h.1615sutikt7e">LZ4 Framing Format</a></span></p><p class="c12 c1"><span class="c3">Frame </span><span class="c3"><a class="c8" href="#h.uof0plru1f66">Descriptor</a></span></p><p class="c1 c12"><span class="c3"><a class="c8" href="#h.u8dkhfnwqyg">Data Blocks</a></span></p><p class="c0"><span class="c3"><a class="c8" href="#h.152pfqac8luc">Skippable </a></span><span class="c3">Frames</span></p><p class="c0"><span class="c3"><a class="c8" href="#h.ujcdmapf87vn">Legacy format</a></span></p><p class="c0"><span class="c3"><a class="c8" href="#h.zij6fhosmkvv">Appendix</a></span></p><p class="c5 c1"><span></span></p><p class="c5 c1"><span class="c6 c11"></span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.1615sutikt7e"></a><span class="c7">General Structure of </span><span class="c7">LZ4 Framing format</span></h1><p class="c5 c9 c1"><span class="c6 c11"></span></p><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 408.00px; height: 106.00px;"><img alt="" src="images/image05.png" style="width: 408.00px; height: 106.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c5 c1"><span class="c6 c11"></span></p><p class="c1"><span class="c6">Magic Number</span></p><p class="c1"><span>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br>Value : </span><span class="c14 c18">0x184D2204</span></p><p class="c5 c1"><span class="c2"></span></p><p class="c1"><span class="c6">Frame D</span><span class="c6">escriptor</span></p><p class="c1"><span>3</span><span> to 1</span><span>1</span><span> Bytes, to be detailed </span><span>in the next part.</span><span><br>Most important part of the spec.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Data Blocks</span></p><p class="c1"><span>To be detailed later on.<br>That’s where compressed data is stored.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">EndMark</span></p><p class="c1"><span>The flow of </span><span>blocks </span><span>ends when the last data block has a size of “</span><span class="c14">0</span><span>”. </span><span><br></span><span>The size is expressed as </span><span>a </span><span>32-bits value.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Content Checksum</span></p><p class="c1"><span>Content Checksum verify that the full content has been decoded correctly.<br>The content checksum is the result of </span><span class="c3"><a class="c8" href="http://code.google.com/p/xxhash/">xxh32()</a></span><span> hash function digesting the original (decoded) data as input, and a seed of zero.<br>Content checksum is only present when its </span><span class="c3"><a class="c8" href="#id.s5zerkv6retr">associated flag </a></span><span>is set in the framing descriptor. Content Checksum validates the result, that all blocks were fully transmitted in the correct order and without error, and also that the encoding/decoding process itself generated no distortion. Its usage is recommended. </span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Frame Concatenation</span></p><p class="c1"><span>In some circumstances, it may be preferable </span><span>to append multiple frames, </span><span>for example </span><span>in order to add new data to an existing compressed file without re-framing it.</span></p><p class="c1"><span>In such case, each frame has its own set of descriptor flags. Each frame is considered independent. The only relation between frames is their sequential order.</span></p><p class="c1"><span>The ability to decode multiple concatenated frames within a single stream or file is left outside of this specification. As an example, the reference lz4 command line utility behavior is to decode all concatenated frames in their sequential order. </span></p><p class="c5 c1"><span></span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span></span></p><h2 class="c1"><a name="h.uof0plru1f66"></a><span class="c7">Frame </span><span class="c7">Descriptor</span></h2><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 226.00px; height: 106.00px;"><img alt="" src="images/image00.png" style="width: 226.00px; height: 106.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 606.00px; height: 125.33px;"><img alt="" src="images/image01.png" style="width: 606.00px; height: 125.33px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>The descriptor uses a minimum of </span><span>3</span><span> bytes</span><span>, and up to 11 bytes depending on optional parameters.</span><span><br>In the picture, bit 7 is highest bit, while bit 0 is lowest.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Version Number :</span></p><p class="c1"><span>2-bits field, </span><span class="c6">must</span><span class="c14"> </span><span>be set to “</span><span class="c14">01</span><span>”.<br>Any other value cannot be decoded by this </span><span>version of the specification.</span><span><br>Other version numbers will use different flag layouts.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Block </span><span class="c6">Independence </span><span class="c6">flag :</span></p><p class="c1"><span>If this flag is set to “1”</span><span>, blocks are independent, and can therefore be decoded independently, in parallel.<br>If this flag is set to “</span><span>0</span><span>”, each block depends on previous ones for decoding (up to LZ4 window size, which is 64 KB). In this case, it’s necessary to decode all blocks in sequence.</span></p><p class="c1"><span>Block </span><span>dependency</span><span> improves compression ratio, especially for small blocks. On the other hand, it makes jumps or multi-threaded decoding impossible.</span></p><p class="c5 c1"><span></span></p><a href="#" name="id.r4mqxzdxswxz"></a><p class="c1"><span class="c6">Block checksum flag :</span></p><p class="c1"><span>If this flag is set, e</span><span>ach data block will be followed by a 4-bytes checksum, calculated by using the xxHash-32 algorithm on the raw (compressed) data block.<br>The intention is to detect data corruption (storage or transmission errors) immediately, before decoding.<br>Block ch</span><span>ecksum usage is optional.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Content </span><span class="c6">Size flag :</span></p><p class="c1"><span>If this flag is set, the original (uncompressed) size of data included within </span><span>the frame</span><span> will be present as an 8 bytes unsigned value, litt</span><span>le endian format, </span><span>after the flags.</span></p><p class="c1"><span>Recommended </span><span>value : “</span><span class="c14">0</span><span>” (not present)</span></p><p class="c1 c5"><span></span></p><p class="c1"><span class="c6">Content checksum flag :</span></p><p class="c1"><span>If this flag is set, a</span><span class="c3"><a class="c8" href="#id.q3958klk497z"> content checksum</a></span><span> will be appended after the EoS mark.</span></p><p class="c1"><span>Recommended value : “</span><span class="c14">1</span><span>” (content checksum is present)</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Block Maximum Size :</span></p><p class="c1"><span>This information is intended to help the decoder allocate the right amount of memory.<br>Size here refers to the original (uncompressed) data size.<br>Block Maximum Size </span><span>is</span><span> one value among the fol</span><span>lowing table : </span></p><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 331.00px; height: 56.00px;"><img alt="" src="images/image03.png" style="width: 331.00px; height: 56.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>The decoder may refuse to allocate block sizes above a (system-specific) size.<br>Unused values may be used in a future revision of the spec.<br>A decoder conformant to the current version of the spec is only able to decode blocksizes defined in this spec.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Reserved bits :</span></p><p class="c1"><span>Value of reserved bits </span><span class="c6">must </span><span>be </span><span class="c14">0</span><span> (zero).<br>Reserved bit might be used in a future version of the specification, to enable any (yet-to-decide) optional feature.<br>If this happens, a decoder respecting the current version of the specification shall not be able to decode such a frame.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Content Size</span></p><p class="c1"><span>This is the original (uncompressed) size. <br>This information is optional, and only present if the </span><span class="c3"><a class="c8" href="#id.tqyy099hxhnn">associated flag is set</a></span><span>.<br>Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes.<br>Format is Little endian.<br>This field has no impact on decoding, it just informs the decoder how much data the frame holds (for example, to display it during decoding process, or for verification purpose). It can be safely skipped by a conformant decoder.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Header Checksum :</span></p><p class="c1"><span>One-byte checksum of all descriptor fields, including optional ones when present.<br>The byte is second byte of </span><span class="c3"><a class="c8" href="http://code.google.com/p/xxhash/">xxh32()</a></span><span> : { (xxh32()>>8) & 0xFF } ,<br>using zero as a seed, <br>and the full Frame Descriptor as an input (</span><span class="c7 c15">including</span><span> optional fields when they are present).<br>A different checksum indicates an error in the descriptor.</span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h2 class="c1"><a name="h.u8dkhfnwqyg"></a><span class="c7">Data Blocks</span></h2><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 317.00px; height: 90.00px;"><img alt="" src="images/image02.png" style="width: 317.00px; height: 90.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c5 c1"><span class="c6 c11"></span></p><p class="c1"><span class="c6">Block</span><span class="c6"> Size</span></p><p class="c1"><span>Th</span><span>is</span><span> field uses </span><span class="c14">4</span><span class="c14">-bytes, </span><span>f</span><span>ormat is </span><span class="c7">little-endian</span><span>.</span></p><p class="c1"><span>The highest bit is “</span><span class="c14">1</span><span>” if data in the block is uncompressed.</span></p><p class="c1"><span>The highest bit is “</span><span class="c14">0</span><span>” if data in the block is compressed by LZ4.</span></p><p class="c1"><span>All other bits give the size, in bytes, of the following data block (the size does not include the checksum if present).</span></p><p class="c1"><span>Block Size shall never be larger than Block Maximum Size. Such a thing could happen when the original data is incompressible. In this case, such a data block shall be passed in uncompressed format.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Data</span></p><p class="c1"><span>Where the actual data to decode stands. It might be compressed or not, depending on previous field indications.<br>Uncompressed size of Data can be any size, up to “block maximum size”. <br>Note that data block is not necessarily </span><span>full </span><span>: an arbitrary “flush” may happen anytime. Any block can be </span><span>“partially filled”.</span></p><p class="c5 c1"><span></span></p><a href="#" name="id.3p4pcqe6ab8n"></a><p class="c1"><span class="c6">Block checksum :</span></p><p class="c1"><span>Only present if the </span><span class="c3"><a class="c8" href="#id.r4mqxzdxswxz">associated flag is set</a></span><span>.<br>This is a 4-bytes checksum value, in little endian format, <br>calculated by using the xxHash-32 algorithm </span><span class="c7">on the raw (undecoded) data block</span><span>, <br>and a seed of zero.</span><span><br>The intention is to detect data corruption (storage or transmission errors) </span><span class="c15">before </span><span>decoding.</span></p><p class="c1"><span>Block checksum is cumulative with Content checksum.</span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.152pfqac8luc"></a><span class="c7">Skippable Frames</span></h1><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 285.00px; height: 106.00px;"><img alt="LZ4 Framing Format - Skippable Frame.png" src="images/image04.png" style="width: 285.00px; height: 106.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>Skippable frames allow the integration of user-defined data into a flow of concatenated frames.<br>Its design is pretty straightforward, with the sole objective to allow the decoder to quickly skip over user-defined data and continue decoding.</span></p><p class="c1"><span>For the purpose of facilitating identification, it is discouraged to start a flow of concatenated frames with a skippable frame. If there is a need to start such a flow with some user data encapsulated into a skippable frame, it’s recommended to start will a zero-byte LZ4 frame followed by a skippable frame. This will make it easier for file type identifiers.</span></p><p class="c1"><span> </span></p><p class="c1"><span class="c6">Magic Number</span></p><p class="c1"><span>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br>Value : </span><span class="c2">0x184D2A5X</span><span>, which means any value from</span><span class="c2"> 0x184D2A50 to 0x184D2A5F.</span><span> All 16 values are valid to identify a skippable frame.<br></span></p><p class="c1"><span class="c6">Frame Size</span><span class="c6"> </span></p><p class="c1"><span>This is the size, in bytes, of the following User Data (without including the magic number nor the size field itself).<br>4 Bytes, </span><span class="c7">Little endian</span><span> format, unsigned 32-bits.<br>This means User Data can’t be bigger than (2^32-1) Bytes.<br></span></p><p class="c1"><span class="c6">User Data</span></p><p class="c1"><span>User Data can be anything. Data will just be skipped by the decoder. </span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.ujcdmapf87vn"></a><span class="c7">Legacy frame</span></h1><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 570.00px; height: 90.00px;"><img alt="" src="images/image06.png" style="width: 570.00px; height: 90.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>The Legacy frame format was defined into the initial versions of “LZ4Demo”.<br>Newer compressors should not use this format anymore, since it is too restrictive.<br>It is recommended that decompressors shall be able to decode this format during the transition period.</span></p><p class="c1"><span>Main properties of legacy format :<br>- Fixed block size : </span><span>8 MB</span><span>.<br>- All blocks must be completely filled, except the last one.<br>- All blocks are always compressed, even when compression is detri</span><span>mental.</span><span><br>- The last block is detected either because it is followed by the “EOF” (End of File) mark</span><span>, or because it is followed by a known Frame Magic Number.</span><span><br>- No checksum<br>- Convention is Little endian</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Magic Number</span></p><p class="c1"><span>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br>Value : </span><span class="c2">0x184C2102<br></span></p><p class="c1"><span class="c6">Block Compressed Size</span></p><p class="c1"><span>This is the size, in bytes, of the following compressed data block.<br>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br></span></p><p class="c1"><span class="c6">Data</span></p><p class="c1"><span>Where the actual data stands. <br>Data is </span><span class="c7">always</span><span> compressed, even when compression is detrimental (i.e. larger than original size).</span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.zij6fhosmkvv"></a><span class="c6">Appendix </span><span> </span></h1><p class="c1"><span class="c11">Version changes</span></p><p class="c1"><span>1.4.1 : changed wording from “stream” to “frame”</span></p><p class="c1"><span>1.4 : added skippable streams, re-added stream checksum </span></p><p class="c1"><span>1.3 : modified header checksum</span></p><p class="c1"><span>1.2 : reduced choice of “block size”, to postpone decision on “dynamic size of BlockSize Field”.</span></p><p class="c1"><span>1.1 : optional fields are now part of the descriptor</span></p><p class="c1"><span>1.0 : changed “block size” specification, adding a compressed/uncompressed flag</span></p><p class="c1"><span>0.9 : reduced scale of “block maximum size” table</span></p><p class="c1"><span>0.8 : removed : high compression flag</span></p><p class="c1"><span>0.7 : removed : stream checksum</span></p><p class="c1"><span>0.6 : settled : stream size uses 8 bytes, endian convention is little endian</span></p><p class="c1"><span>0.5: added copyright notice</span></p><p class="c1"><span>0.4 : changed format to Google Doc compatible OpenDocument</span></p></body></html>
\ No newline at end of file @@ -31,7 +31,7 @@ # ################################################################ # Version number -export VERSION=128 +export VERSION=129 export RELEASE=r$(VERSION) DESTDIR?= @@ -41,24 +41,6 @@ LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include PRGDIR = programs LZ4DIR = lib -DISTRIBNAME=lz4-$(RELEASE).tar.gz - -TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \ - $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h \ - $(LZ4DIR)/xxhash.c $(LZ4DIR)/xxhash.h \ - $(LZ4DIR)/liblz4.pc.in $(LZ4DIR)/Makefile $(LZ4DIR)/LICENSE \ - Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \ - cmake_unofficial/CMakeLists.txt \ - $(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \ - $(PRGDIR)/datagen.c $(PRGDIR)/datagen.h $(PRGDIR)/datagencli.c $(PRGDIR)/fuzzer.c \ - $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ - $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ - $(PRGDIR)/lz4.1 \ - $(PRGDIR)/Makefile $(PRGDIR)/COPYING -NONTEXT = images/image00.png images/image01.png images/image02.png \ - images/image03.png images/image04.png images/image05.png \ - images/image06.png -SOURCES = $(TEXT) $(NONTEXT) # Select test target for Travis CI's Build Matrix @@ -86,7 +68,6 @@ lz4programs: @cd $(PRGDIR); $(MAKE) -e clean: - @rm -f $(DISTRIBNAME) *.sha1 > $(VOID) @cd $(PRGDIR); $(MAKE) clean > $(VOID) @cd $(LZ4DIR); $(MAKE) clean > $(VOID) @cd examples; $(MAKE) clean > $(VOID) @@ -108,24 +89,6 @@ uninstall: travis-install: sudo $(MAKE) install -dist: clean - @install -dD -m 700 lz4-$(RELEASE)/lib/ - @install -dD -m 700 lz4-$(RELEASE)/programs/ - @install -dD -m 700 lz4-$(RELEASE)/cmake_unofficial/ - @install -dD -m 700 lz4-$(RELEASE)/images/ - @for f in $(TEXT); do \ - tr -d '\r' < $$f > .tmp; \ - install -m 600 .tmp lz4-$(RELEASE)/$$f; \ - done - @rm .tmp - @for f in $(NONTEXT); do \ - install -m 600 $$f lz4-$(RELEASE)/$$f; \ - done - @tar -czf $(DISTRIBNAME) lz4-$(RELEASE)/ - @rm -rf lz4-$(RELEASE) - @sha1sum $(DISTRIBNAME) > $(DISTRIBNAME).sha1 - @echo Distribution $(DISTRIBNAME) built - test: @cd $(PRGDIR); $(MAKE) -e test @@ -134,18 +97,21 @@ test-travis: $(TRAVIS_TARGET) cmake: @cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE) +gpptest: clean + $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + clangtest: clean - export CC=clang; $(MAKE) all + $(MAKE) all CC=clang CPPFLAGS="-Werror -Wconversion -Wno-sign-conversion" -staticAnalyze: clean - export CFLAGS=-g; scan-build -v $(MAKE) all +sanitize: clean + $(MAKE) test CC=clang CPPFLAGS="-g -fsanitize=undefined" FUZZER_TIME="-T1mn" NB_LOOPS=-i1 -gpptest: clean - export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all +staticAnalyze: clean + scan-build --status-bugs -v $(MAKE) all CFLAGS=-g armtest: clean - export CC=arm-linux-gnueabi-gcc; cd lib; $(MAKE) -e all - export CC=arm-linux-gnueabi-gcc; cd programs; $(MAKE) -e bins + cd lib; $(MAKE) -e all CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror" + cd programs; $(MAKE) -e bins CC=arm-linux-gnueabi-gcc CPPFLAGS="-Werror" streaming-examples: cd examples; $(MAKE) -e test @@ -1,19 +1,31 @@ +r129: +Added : LZ4_compress_fast() +Changed: New lz4 and lz4hc compression API. Previous function prototypes still supported. +Changed: Sparse file support enabled by default +New : LZ4 CLI improved performance compressing/decompressing multiple files (#86, kind contribution from Kyle J. Harper & Takayuki Matsuoka) +Fixed : GCC 4.9+ optimization bug - Reported by Markus Trippelsdorf, Greg Slazinski & Evan Nemerson +Changed: Enums converted to LZ4F_ namespace convention - by Takayuki Matsuoka +Added : AppVeyor CI environment, for Visual tests - Suggested by Takayuki Matsuoka +Modified:Obsolete functions generate warnings - Suggested by Evan Nemerson, contributed by Takayuki Matsuoka +Fixed : Bug #75 (unfinished stream), reported by Yongwoon Cho +Updated: Documentation converted to MarkDown format + r128: -New : lz4cli sparse file support -New : command -m, to compress multiple files in a single command -Fixed : Restored lz4hc compression ratio (was slightly lower since r124) -New : lz4 cli supports long commands -New : lz4frame & lz4cli frame content size support -New : lz4frame supports skippable frames -Changed:Default "make install" directory is /usr/local -New : lz4 cli supports "pass-through" mode -New : datagen can generate sparse files -New : scan-build tests -New : g++ compatibility tests -New : arm cross-compilation test -Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48) -Added : Visual project directory -Updated:Man page & Specification +New : lz4cli sparse file support (Requested by Neil Wilson, and contributed by Takayuki Matsuoka) +New : command -m, to compress multiple files in a single command (suggested by Kyle J. Harper) +Fixed : Restored lz4hc compression ratio (slightly lower since r124) +New : lz4 cli supports long commands (suggested by Takayuki Matsuoka) +New : lz4frame & lz4cli frame content size support +New : lz4frame supports skippable frames, as requested by Sergey Cherepanov +Changed: Default "make install" directory is /usr/local, as notified by Ron Johnson +New : lz4 cli supports "pass-through" mode, requested by Neil Wilson +New : datagen can generate sparse files +New : scan-build tests, thanks to kind help by Takayuki Matsuoka +New : g++ compatibility tests +New : arm cross-compilation test, thanks to kind help by Takayuki Matsuoka +Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48, reported by Thomas Klausner) +Added : Visual project directory +Updated: Man page & Specification r127: N/A : added a file on SVN @@ -1,60 +1,80 @@ LZ4 - Extremely fast compression ================================ -LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. -A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio. +LZ4 is lossless compression algorithm, +providing compression speed at 400 MB/s per core, +scalable with multi-cores CPU. +It also features an extremely fast decoder, +with speed in multiple GB/s per core, +typically reaching RAM speed limits on multi-core systems. + +Speed can be tuned dynamically, selecting an "acceleration" factor +which trades compression ratio for more speed up. +On the other end, a high compression derivative, LZ4_HC, is also provided, +trading CPU time for improved compression ratio. +All versions feature the same excellent decompression speed. + |Branch |Status | |------------|---------| -|master | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=master)](https://travis-ci.org/Cyan4973/lz4) | -|dev | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=dev)](https://travis-ci.org/Cyan4973/lz4) | +|master | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=master)](https://travis-ci.org/Cyan4973/lz4) [![Build status](https://ci.appveyor.com/api/projects/status/v6kxv9si529477cq/branch/master?svg=true)](https://ci.appveyor.com/project/YannCollet/lz4) | +|dev | [![Build Status](https://travis-ci.org/Cyan4973/lz4.svg?branch=dev)](https://travis-ci.org/Cyan4973/lz4) [![Build status](https://ci.appveyor.com/api/projects/status/v6kxv9si529477cq/branch/dev?svg=true)](https://ci.appveyor.com/project/YannCollet/lz4) | + + > **Branch Policy:** > - The "master" branch is considered stable, at all times. -> - The "dev" branch is the one where all contributions must be merged before being promoted to master. -> - If you plan to propose a patch, please commit into the "dev" branch. Direct commit to "master" are not permitted. -> - Feature branches can also exist, for dedicated testing of larger modifications before merge into "dev" branch. +> - The "dev" branch is the one where all contributions must be merged + before being promoted to master. +> + If you plan to propose a patch, please commit into the "dev" branch, + or its own feature branch. + Direct commit to "master" are not permitted. Benchmarks ------------------------- -The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.2)](http://encode.ru/threads/1371-Filesystem-benchmark?p=33548&viewfull=1#post33548) compiled with GCC v4.6.1 on Linux Ubuntu 64-bits v11.10, -The reference system uses a Core i5-3340M @2.7GHz. -Benchmark evaluates the compression of reference [Silesia Corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia) in single-thread mode. - -<table> - <tr> - <th>Compressor</th><th>Ratio</th><th>Compression</th><th>Decompression</th> - </tr> - <tr> - <th>LZ4 (r101)</th><th>2.084</th><th>422 MB/s</th><th>1820 MB/s</th> - </tr> - <tr> - <th>LZO 2.06</th><th>2.106</th><th>414 MB/s</th><th>600 MB/s</th> - </tr> - <tr> - <th>QuickLZ 1.5.1b6</th><th>2.237</th><th>373 MB/s</th><th>420 MB/s</th> - </tr> - <tr> - <th>Snappy 1.1.0</th><th>2.091</th><th>323 MB/s</th><th>1070 MB/s</th> - </tr> - <tr> - <th>LZF</th><th>2.077</th><th>270 MB/s</th><th>570 MB/s</th> - </tr> - <tr> - <th>zlib 1.2.8 -1</th><th>2.730</th><th>65 MB/s</th><th>280 MB/s</th> - </tr> - <tr> - <th>LZ4 HC (r101)</th><th>2.720</th><th>25 MB/s</th><th>2080 MB/s</th> - </tr> - <tr> - <th>zlib 1.2.8 -6</th><th>3.099</th><th>21 MB/s</th><th>300 MB/s</th> - </tr> -</table> - -The LZ4 block compression format is detailed within [lz4_block_format.txt](lz4_block_format.txt). - -For streaming unknown amount of data and compress files of any size, a frame format has been published, and can be consulted within the file LZ4_Frame_Format.html . +The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.3)] +compiled with GCC v4.8.2 on Linux Mint 64-bits v17. +The reference system uses a Core i5-4300U @1.9GHz. +Benchmark evaluates the compression of reference [Silesia Corpus] +in single-thread mode. + +| Compressor | Ratio | Compression | Decompression | +| ---------- | ----- | ----------- | ------------- | +| memcpy | 1.000 | 4200 MB/s | 4200 MB/s | +|**LZ4 fast 17 (r129)**| 1.607 |**690 MB/s** | **2220 MB/s** | +|**LZ4 default (r129)**|**2.101**|**385 MB/s** | **1850 MB/s** | +| LZO 2.06 | 2.108 | 350 MB/s | 510 MB/s | +| QuickLZ 1.5.1.b6 | 2.238 | 320 MB/s | 380 MB/s | +| Snappy 1.1.0 | 2.091 | 250 MB/s | 960 MB/s | +| zlib 1.2.8 -1 | 2.730 | 59 MB/s | 250 MB/s | +|**LZ4 HC (r129)** |**2.720**| 22 MB/s | **1830 MB/s** | +| zlib 1.2.8 -6 | 3.099 | 18 MB/s | 270 MB/s | + + +Documentation +------------------------- + +The raw LZ4 block compression format is detailed within [lz4_Block_format]. + +To compress an arbitrarily long file or data stream, multiple blocks are required. +Organizing these blocks and providing a common header format to handle their content +is the purpose of the Frame format, defined into [lz4_Frame_format]. +Interoperable versions of LZ4 must respect this frame format. + + +Other source versions +------------------------- + +Beyond the C reference source, +many contributors have created versions of lz4 in multiple languages. +A list of these sources is maintained on the [LZ4 Homepage]. + +[Open-Source Benchmark program by m^2 (v0.14.3)]: http://encode.ru/threads/1371-Filesystem-benchmark?p=34029&viewfull=1#post34029 +[Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia +[lz4_Block_format]: lz4_Block_format.md +[lz4_Frame_format]: lz4_Frame_format.md +[LZ4 Homepage]: http://www.lz4.info diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt index 1782ea0..4c3eb65 100644 --- a/cmake_unofficial/CMakeLists.txt +++ b/cmake_unofficial/CMakeLists.txt @@ -17,13 +17,15 @@ ENDIF() option(BUILD_TOOLS "Build the command line tools" ON) option(BUILD_LIBS "Build the libraries in addition to the tools" ON) +if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) if(UNIX AND BUILD_LIBS) add_definitions(-fPIC) endif() +endif() set(LZ4_DIR ../lib/) set(PRG_DIR ../programs/) -set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c) +set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h ${LZ4_DIR}lz4frame.c ${LZ4_DIR}lz4frame.h ${LZ4_DIR}xxhash.c) set(LZ4_SRCS ${LZ4_DIR}lz4frame.c ${LZ4_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c ${PRG_DIR}lz4io.c) if(BUILD_TOOLS AND NOT BUILD_LIBS) @@ -51,6 +53,7 @@ if(BUILD_LIBS) install(FILES ${LZ4_DIR}/lz4.h ${LZ4_DIR}/lz4hc.h + ${LZ4_DIR}/lz4frame.h DESTINATION include ) @@ -62,7 +65,12 @@ endif() #warnings +if(MSVC) +ADD_DEFINITIONS("-W4") +endif() +if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) ADD_DEFINITIONS("-Wall") +endif() if(CMAKE_COMPILER_IS_GNUCXX) ADD_DEFINITIONS("-Wextra") ADD_DEFINITIONS("-Wundef") @@ -70,7 +78,10 @@ ADD_DEFINITIONS("-Wshadow") ADD_DEFINITIONS("-Wcast-align") ADD_DEFINITIONS("-Wstrict-prototypes") endif(CMAKE_COMPILER_IS_GNUCXX) +if((CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) AND + (NOT CMAKE_SYSTEM_NAME MATCHES "SunOS")) ADD_DEFINITIONS("-std=c99") +endif() ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"") INCLUDE_DIRECTORIES (${LZ4_DIR}) diff --git a/examples/Makefile b/examples/Makefile index 0c4cf13..808b511 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -1,6 +1,7 @@ # ########################################################################## # LZ4 examples - Makefile # Copyright (C) Yann Collet 2011-2014 +# # GPL v2 License # # This program is free software; you can redistribute it and/or modify @@ -21,29 +22,17 @@ # - LZ4 source repository : http://code.google.com/p/lz4/ # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c # ########################################################################## -# lz4 : Command Line Utility, supporting gzip-like arguments -# lz4c : CLU, supporting also legacy lz4demo arguments -# lz4c32: Same as lz4c, but forced to compile in 32-bits mode -# fuzzer : Test tool, to check lz4 integrity on target platform -# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode -# fullbench : Precisely measure speed for each LZ4 function variant -# fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# This makefile compile and test +# example programs, using (mostly) LZ4 streaming library, +# kindly provided by Takayuki Matsuoka # ########################################################################## -CC := $(CC) CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -Wno-missing-braces # Wno-missing-braces required due to GCC <4.8.3 bug -FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes +FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) TESTFILE= Makefile -LZ4DIR = ../lib - - -# Minimize test target for Travis CI's Build Matrix -ifeq ($(LZ4_TRAVIS_CI_ENV),-m32) -CFLAGS += -m32 -else ifeq ($(LZ4_TRAVIS_CI_ENV),-m64) -endif +LZ4DIR := ../lib # Define *.exe as extension for Windows systems diff --git a/examples/blockStreaming_doubleBuffer.c b/examples/blockStreaming_doubleBuffer.c index 0adf6ae..efe6fc6 100644 --- a/examples/blockStreaming_doubleBuffer.c +++ b/examples/blockStreaming_doubleBuffer.c @@ -2,7 +2,10 @@ // Copyright : Takayuki Matsuoka -#define _CRT_SECURE_NO_WARNINGS // for MSVC +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS +# define snprintf sprintf_s +#endif #include "lz4.h" #include <stdio.h> @@ -35,11 +38,13 @@ size_t read_bin(FILE* fp, void* array, size_t arrayBytes) { void test_compress(FILE* outFp, FILE* inpFp) { - LZ4_stream_t lz4Stream_body = { 0 }; + LZ4_stream_t lz4Stream_body; LZ4_stream_t* lz4Stream = &lz4Stream_body; char inpBuf[2][BLOCK_BYTES]; int inpBufIndex = 0; + + LZ4_resetStream(lz4Stream); for(;;) { char* const inpPtr = inpBuf[inpBufIndex]; @@ -50,8 +55,8 @@ void test_compress(FILE* outFp, FILE* inpFp) { char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)]; - const int cmpBytes = LZ4_compress_continue( - lz4Stream, inpPtr, cmpBuf, inpBytes); + const int cmpBytes = LZ4_compress_fast_continue( + lz4Stream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf), 1); if(cmpBytes <= 0) { break; } @@ -68,12 +73,14 @@ void test_compress(FILE* outFp, FILE* inpFp) void test_decompress(FILE* outFp, FILE* inpFp) { - LZ4_streamDecode_t lz4StreamDecode_body = { 0 }; + LZ4_streamDecode_t lz4StreamDecode_body; LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; char decBuf[2][BLOCK_BYTES]; int decBufIndex = 0; + LZ4_setStreamDecode(lz4StreamDecode, NULL, 0); + for(;;) { char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)]; int cmpBytes = 0; diff --git a/examples/blockStreaming_lineByLine.c b/examples/blockStreaming_lineByLine.c index 6d14801..f449aa3 100644 --- a/examples/blockStreaming_lineByLine.c +++ b/examples/blockStreaming_lineByLine.c @@ -2,7 +2,10 @@ // Copyright : Takayuki Matsuoka -#define _CRT_SECURE_NO_WARNINGS // for MSVC +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS +# define snprintf sprintf_s +#endif #include "lz4.h" #include <stdio.h> @@ -38,8 +41,9 @@ static void test_compress( size_t ringBufferBytes) { LZ4_stream_t* const lz4Stream = LZ4_createStream(); - char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); - char* const inpBuf = malloc(ringBufferBytes); + const size_t cmpBufBytes = LZ4_COMPRESSBOUND(messageMaxBytes); + char* const cmpBuf = (char*) malloc(cmpBufBytes); + char* const inpBuf = (char*) malloc(ringBufferBytes); int inpOffset = 0; for ( ; ; ) @@ -60,8 +64,8 @@ static void test_compress( #endif { - const int cmpBytes = LZ4_compress_continue( - lz4Stream, inpPtr, cmpBuf, inpBytes); + const int cmpBytes = LZ4_compress_fast_continue( + lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1); if (cmpBytes <= 0) break; write_uint16(outFp, (uint16_t) cmpBytes); write_bin(outFp, cmpBuf, cmpBytes); @@ -86,8 +90,8 @@ static void test_decompress( size_t ringBufferBytes) { LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode(); - char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); - char* const decBuf = malloc(ringBufferBytes); + char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes)); + char* const decBuf = (char*) malloc(ringBufferBytes); int decOffset = 0; for ( ; ; ) @@ -121,8 +125,8 @@ static int compare(FILE* f0, FILE* f1) { int result = 0; const size_t tempBufferBytes = 65536; - char* const b0 = malloc(tempBufferBytes); - char* const b1 = malloc(tempBufferBytes); + char* const b0 = (char*) malloc(tempBufferBytes); + char* const b1 = (char*) malloc(tempBufferBytes); while(0 == result) { diff --git a/images/image00.png b/images/image00.png Binary files differdeleted file mode 100644 index f4645f2..0000000 --- a/images/image00.png +++ /dev/null diff --git a/images/image01.png b/images/image01.png Binary files differdeleted file mode 100644 index 661c465..0000000 --- a/images/image01.png +++ /dev/null diff --git a/images/image02.png b/images/image02.png Binary files differdeleted file mode 100644 index a909353..0000000 --- a/images/image02.png +++ /dev/null diff --git a/images/image03.png b/images/image03.png Binary files differdeleted file mode 100644 index 8e1b40c..0000000 --- a/images/image03.png +++ /dev/null diff --git a/images/image04.png b/images/image04.png Binary files differdeleted file mode 100644 index e82d02b..0000000 --- a/images/image04.png +++ /dev/null diff --git a/images/image05.png b/images/image05.png Binary files differdeleted file mode 100644 index 3a8006a..0000000 --- a/images/image05.png +++ /dev/null diff --git a/images/image06.png b/images/image06.png Binary files differdeleted file mode 100644 index 69b4ee6..0000000 --- a/images/image06.png +++ /dev/null diff --git a/lib/Makefile b/lib/Makefile index 8aa15de..4be1499 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -31,7 +31,7 @@ # ################################################################ # Version numbers -VERSION ?= 128 +VERSION ?= 129 LIBVER_MAJOR=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` LIBVER_MINOR=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` LIBVER_PATCH=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lz4.h` @@ -40,7 +40,7 @@ LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) DESTDIR?= PREFIX ?= /usr/local CFLAGS ?= -O3 -CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic +CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wcast-qual -Wstrict-prototypes -pedantic LIBDIR?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include @@ -34,7 +34,7 @@ /************************************** - Tuning parameters +* Tuning parameters **************************************/ /* * HEAPMODE : @@ -44,51 +44,16 @@ #define HEAPMODE 0 /* - * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : - * By default, the source code expects the compiler to correctly optimize - * 4-bytes and 8-bytes read on architectures able to handle it efficiently. - * This is not always the case. In some circumstances (ARM notably), - * the compiler will issue cautious code even when target is able to correctly handle unaligned memory accesses. - * - * You can force the compiler to use unaligned memory access by uncommenting the line below. - * One of the below scenarios will happen : - * 1 - Your target CPU correctly handle unaligned access, and was not well optimized by compiler (good case). - * You will witness large performance improvements (+50% and up). - * Keep the line uncommented and send a word to upstream (https://groups.google.com/forum/#!forum/lz4c) - * The goal is to automatically detect such situations by adding your target CPU within an exception list. - * 2 - Your target CPU correctly handle unaligned access, and was already already optimized by compiler - * No change will be experienced. - * 3 - Your target CPU inefficiently handle unaligned access. - * You will experience a performance loss. Comment back the line. - * 4 - Your target CPU does not handle unaligned access. - * Program will crash. - * If uncommenting results in better performance (case 1) - * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) - * This way, an automatic detection macro can be added to match your case within later versions of the library. + * ACCELERATION_DEFAULT : + * Select the value of "acceleration" for LZ4_compress_fast() when parameter == 0 */ -/* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ +#define ACCELERATION_DEFAULT 1 /************************************** - CPU Feature Detection +* CPU Feature Detection **************************************/ /* - * Automated efficient unaligned memory access detection - * Based on known hardware architectures - * This list will be updated thanks to feedbacks - */ -#if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ - || defined(__ARM_FEATURE_UNALIGNED) \ - || defined(__i386__) || defined(__x86_64__) \ - || defined(_M_IX86) || defined(_M_X64) \ - || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \ - || (defined(_M_ARM) && (_M_ARM >= 7)) -# define LZ4_UNALIGNED_ACCESS 1 -#else -# define LZ4_UNALIGNED_ACCESS 0 -#endif - -/* * LZ4_FORCE_SW_BITCOUNT * Define this parameter if your target system or compiler does not support hardware bit count */ @@ -98,14 +63,14 @@ /************************************** -* Compiler Options +* Includes **************************************/ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -/* "restrict" is a known keyword */ -#else -# define restrict /* Disable restrict */ -#endif +#include "lz4.h" + +/************************************** +* Compiler Options +**************************************/ #ifdef _MSC_VER /* Visual Studio */ # define FORCE_INLINE static __forceinline # include <intrin.h> @@ -113,7 +78,7 @@ # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #else # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# ifdef __GNUC__ +# if defined(__GNUC__) || defined(__clang__) # define FORCE_INLINE static inline __attribute__((always_inline)) # else # define FORCE_INLINE static inline @@ -123,9 +88,8 @@ # endif /* __STDC_VERSION__ */ #endif /* _MSC_VER */ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +/* LZ4_GCC_VERSION is defined into lz4.h */ +#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) @@ -136,7 +100,7 @@ /************************************** - Memory routines +* Memory routines **************************************/ #include <stdlib.h> /* malloc, calloc, free */ #define ALLOCATOR(n,s) calloc(n,s) @@ -146,13 +110,7 @@ /************************************** - Includes -**************************************/ -#include "lz4.h" - - -/************************************** - Basic Types +* Basic Types **************************************/ #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include <stdint.h> @@ -171,7 +129,7 @@ /************************************** - Reading and writing into memory +* Reading and writing into memory **************************************/ #define STEPSIZE sizeof(size_t) @@ -184,10 +142,19 @@ static unsigned LZ4_isLittleEndian(void) } +static U16 LZ4_read16(const void* memPtr) +{ + U16 val16; + memcpy(&val16, memPtr, 2); + return val16; +} + static U16 LZ4_readLE16(const void* memPtr) { - if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) - return *(U16*)memPtr; + if (LZ4_isLittleEndian()) + { + return LZ4_read16(memPtr); + } else { const BYTE* p = (const BYTE*)memPtr; @@ -197,10 +164,9 @@ static U16 LZ4_readLE16(const void* memPtr) static void LZ4_writeLE16(void* memPtr, U16 value) { - if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) + if (LZ4_isLittleEndian()) { - *(U16*)memPtr = value; - return; + memcpy(memPtr, &value, 2); } else { @@ -210,41 +176,18 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } } - -static U16 LZ4_read16(const void* memPtr) -{ - if (LZ4_UNALIGNED_ACCESS) - return *(U16*)memPtr; - else - { - U16 val16; - memcpy(&val16, memPtr, 2); - return val16; - } -} - static U32 LZ4_read32(const void* memPtr) { - if (LZ4_UNALIGNED_ACCESS) - return *(U32*)memPtr; - else - { - U32 val32; - memcpy(&val32, memPtr, 4); - return val32; - } + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; } static U64 LZ4_read64(const void* memPtr) { - if (LZ4_UNALIGNED_ACCESS) - return *(U64*)memPtr; - else - { - U64 val64; - memcpy(&val64, memPtr, 8); - return val64; - } + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; } static size_t LZ4_read_ARCH(const void* p) @@ -256,31 +199,9 @@ static size_t LZ4_read_ARCH(const void* p) } -static void LZ4_copy4(void* dstPtr, const void* srcPtr) -{ - if (LZ4_UNALIGNED_ACCESS) - { - *(U32*)dstPtr = *(U32*)srcPtr; - return; - } - memcpy(dstPtr, srcPtr, 4); -} +static void LZ4_copy4(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 4); } -static void LZ4_copy8(void* dstPtr, const void* srcPtr) -{ -#if GCC_VERSION!=409 /* disabled on GCC 4.9, as it generates invalid opcode (crash) */ - if (LZ4_UNALIGNED_ACCESS) - { - if (LZ4_64bits()) - *(U64*)dstPtr = *(U64*)srcPtr; - else - ((U32*)dstPtr)[0] = ((U32*)srcPtr)[0], - ((U32*)dstPtr)[1] = ((U32*)srcPtr)[1]; - return; - } -#endif - memcpy(dstPtr, srcPtr, 8); -} +static void LZ4_copy8(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 8); } /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) @@ -293,7 +214,7 @@ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) /************************************** - Common Constants +* Common Constants **************************************/ #define MINMATCH 4 @@ -334,7 +255,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; @@ -347,7 +268,7 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r; _BitScanForward( &r, (U32)val ); return (int)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; @@ -363,8 +284,8 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanReverse64( &r, val ); return (unsigned)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); # else unsigned r; if (!(val>>32)) { r=4; } else { r=0; val>>=32; } @@ -379,8 +300,8 @@ static unsigned LZ4_NbCommonBytes (register size_t val) unsigned long r = 0; _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); -# elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } @@ -423,13 +344,6 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /************************************** -* Local Utils -**************************************/ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } - - -/************************************** * Local Structures and types **************************************/ typedef struct { @@ -437,7 +351,7 @@ typedef struct { U32 currentOffset; U32 initCheck; const BYTE* dictionary; - const BYTE* bufferStart; + BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ U32 dictSize; } LZ4_stream_t_internal; @@ -451,6 +365,14 @@ typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { full = 0, partial = 1 } earlyEnd_directive; +/************************************** +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + + /******************************** * Compression functions @@ -464,7 +386,22 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); } +static const U64 prime5bytes = 889523592379ULL; +static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + const U32 hashMask = (1<<hashLog) - 1; + return ((sequence * prime5bytes) >> (40 - hashLog)) & hashMask; +} + +static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType) +{ + if (LZ4_64bits()) + return LZ4_hashSequence64(sequence, tableType); + return LZ4_hashSequence((U32)sequence, tableType); +} + +static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { @@ -496,15 +433,16 @@ static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t t } static int LZ4_compress_generic( - void* ctx, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, - limitedOutput_directive outputLimited, - tableType_t const tableType, - dict_directive dict, - dictIssue_directive dictIssue) + void* const ctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) { LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; @@ -527,7 +465,7 @@ static int LZ4_compress_generic( size_t refDelta=0; /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ switch(dict) { case noDict: @@ -558,15 +496,15 @@ static int LZ4_compress_generic( BYTE* token; { const BYTE* forwardIp = ip; - unsigned step=1; - unsigned searchMatchNb = (1U << LZ4_skipTrigger); + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; /* Find a match */ do { U32 h = forwardH; ip = forwardIp; forwardIp += step; - step = searchMatchNb++ >> LZ4_skipTrigger; + step = (searchMatchNb++ >> LZ4_skipTrigger); if (unlikely(forwardIp > mflimit)) goto _last_literals; @@ -693,13 +631,22 @@ _next_match: _last_literals: /* Encode Last Literals */ { - int lastRun = (int)(iend - anchor); - if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) + const size_t lastRun = (size_t)(iend - anchor); + if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun<<ML_BITS); - memcpy(op, anchor, iend - anchor); - op += iend-anchor; + if (lastRun >= RUN_MASK) + { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRun<<ML_BITS); + } + memcpy(op, anchor, lastRun); + op += lastRun; } /* End */ @@ -707,39 +654,271 @@ _last_literals: } -int LZ4_compress(const char* source, char* dest, int inputSize) +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_resetStream((LZ4_stream_t*)state); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + if (maxOutputSize >= LZ4_compressBound(inputSize)) + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } + else + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } +} + + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { #if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */ + void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else - U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */ + LZ4_stream_t ctx; + void* ctxPtr = &ctx; #endif - int result; - if (inputSize < LZ4_64Klimit) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); - else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); + int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (HEAPMODE) - FREEMEM(ctx); + FREEMEM(ctxPtr); #endif return result; } -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) + +int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); +} + + +/* hidden debug function */ +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t ctx; + + LZ4_resetStream(&ctx); + + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); +} + + +/******************************** +* destSize variant +********************************/ + +static int LZ4_compress_destSize_generic( + void* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* base = (const BYTE*) src; + const BYTE* lowLimit = (const BYTE*) src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + targetDstSize; + BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; + BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); + BYTE* const oMaxSeq = oMaxLit - 1 /* token */; + + U32 forwardH; + + + /* Init conditions */ + if (targetDstSize < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtr<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* First Byte */ + *srcSizePtr = 0; + LZ4_putPosition(ip, ctx, tableType, base); + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) + { + const BYTE* match; + BYTE* token; + { + const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = 1 << LZ4_skipTrigger; + + /* Find a match */ + do { + U32 h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) + goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) + { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + if (litLength>=RUN_MASK) + { + unsigned len = litLength - RUN_MASK; + *token=(RUN_MASK<<ML_BITS); + for(; len >= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<<ML_BITS); + + /* Copy Literals */ + LZ4_wildCopy(op, anchor, op+litLength); + op += litLength; + } + +_next_match: + /* Encode Offset */ + LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + + /* Encode MatchLength */ + { + size_t matchLength; + + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + + if (op + ((matchLength+240)/255) > oMaxMatch) + { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + //printf("offset %5i, matchLength%5i \n", (int)(ip-match), matchLength + MINMATCH); + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) + { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, ctx, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + size_t lastRunSize = (size_t)(iend - anchor); + if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) + { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) + { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRunSize<<ML_BITS); + } + memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip)-src); + return (int) (((char*)op)-dst); +} + + +static int LZ4_compress_destSize_extState (void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ + LZ4_resetStream((LZ4_stream_t*)state); + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) /* compression success is guaranteed */ + { + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } + else + { + if (*srcSizePtr < LZ4_64Klimit) + return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); + } +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */ + void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else - U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */ + LZ4_stream_t ctxBody; + void* ctx = &ctxBody; #endif - int result; - if (inputSize < LZ4_64Klimit) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); - else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); #if (HEAPMODE) FREEMEM(ctx); @@ -748,19 +927,10 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in } -/***************************************** -* Experimental : Streaming functions -*****************************************/ -/* - * LZ4_initStream - * Use this function once, to init a newly allocated LZ4_stream_t structure - * Return : 1 if OK, 0 if error - */ -void LZ4_resetStream (LZ4_stream_t* LZ4_stream) -{ - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); -} +/******************************** +* Streaming functions +********************************/ LZ4_stream_t* LZ4_createStream(void) { @@ -770,6 +940,11 @@ LZ4_stream_t* LZ4_createStream(void) return lz4s; } +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); +} + int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { FREEMEM(LZ4_stream); @@ -777,6 +952,7 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) } +#define HASH_UNIT sizeof(size_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; @@ -784,24 +960,26 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) const BYTE* const dictEnd = p + dictSize; const BYTE* base; - if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */ + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ4_resetStream(LZ4_dict); - if (dictSize < MINMATCH) + if (dictSize < (int)HASH_UNIT) { dict->dictionary = NULL; dict->dictSize = 0; return 0; } - if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB; + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->currentOffset += 64 KB; base = p - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); dict->currentOffset += dict->dictSize; - while (p <= dictEnd-MINMATCH) + while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict, byU32, base); + LZ4_putPosition(p, dict->hashTable, byU32, base); p+=3; } @@ -830,8 +1008,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) } -FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize, - int maxOutputSize, limitedOutput_directive limit) +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; @@ -840,6 +1017,7 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; LZ4_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; /* Check overlapping input/dictionary space */ { @@ -858,9 +1036,9 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); streamPtr->dictSize += (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; @@ -870,9 +1048,9 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; @@ -880,18 +1058,8 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so } } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) -{ - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited); -} - -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput); -} - -/* Hidden debug function, to force separate dictionary mode */ +/* Hidden debug function, to force external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; @@ -902,7 +1070,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); - result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue); + result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; @@ -955,7 +1123,7 @@ FORCE_INLINE int LZ4_decompress_generic( ) { /* Local Variables */ - const BYTE* restrict ip = (const BYTE*) source; + const BYTE* ip = (const BYTE*) source; const BYTE* const iend = ip + inputSize; BYTE* op = (BYTE*) dest; @@ -1051,8 +1219,7 @@ FORCE_INLINE int LZ4_decompress_generic( { /* match can be copied as a single segment from external dictionary */ match = dictEnd - (lowPrefix-match); - memcpy(op, match, length); - op += length; + memmove(op, match, length); op += length; } else { @@ -1110,11 +1277,11 @@ FORCE_INLINE int LZ4_decompress_generic( if (endOnInput) return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ else - return (int) (((char*)ip)-source); /* Nb of input bytes read */ + return (int) (((const char*)ip)-source); /* Nb of input bytes read */ /* Overflow error detected */ _output_error: - return (int) (-(((char*)ip)-source))-1; + return (int) (-(((const char*)ip)-source))-1; } @@ -1138,9 +1305,9 @@ int LZ4_decompress_fast(const char* source, char* dest, int originalSize) typedef struct { - BYTE* externalDict; + const BYTE* externalDict; size_t extDictSize; - BYTE* prefixEnd; + const BYTE* prefixEnd; size_t prefixSize; } LZ4_streamDecode_t_internal; @@ -1172,7 +1339,7 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti { LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; lz4sd->prefixSize = (size_t) dictSize; - lz4sd->prefixEnd = (BYTE*) dictionary + dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; lz4sd->externalDict = NULL; lz4sd->extDictSize = 0; return 1; @@ -1261,7 +1428,7 @@ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) @@ -1277,13 +1444,21 @@ int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSi /* debug function */ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } /*************************************************** * Obsolete Functions ***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } +int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } + /* These function names are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. @@ -1298,23 +1473,23 @@ int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } -static void LZ4_init(LZ4_stream_t_internal* lz4ds, const BYTE* base) +static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base) { MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); lz4ds->bufferStart = base; } -int LZ4_resetStreamState(void* state, const char* inputBuffer) +int LZ4_resetStreamState(void* state, char* inputBuffer) { if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t_internal*)state, (const BYTE*)inputBuffer); + LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer); return 0; } -void* LZ4_create (const char* inputBuffer) +void* LZ4_create (char* inputBuffer) { void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer); + LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer); return lz4ds; } @@ -1325,32 +1500,6 @@ char* LZ4_slideInputBuffer (void* LZ4_Data) return (char*)(ctx->bufferStart + dictSize); } -/* Obsolete compresson functions using User-allocated state */ - -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) -{ - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); -} - -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); -} - /* Obsolete streaming decompression functions */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) @@ -39,16 +39,16 @@ extern "C" { #endif /* - * lz4.h provides block compression functions, for optimal performance. + * lz4.h provides block compression functions, and gives full buffer control to programmer. * If you need to generate inter-operable compressed data (respecting LZ4 frame specification), - * please use lz4frame.h instead. + * and can let the library handle its own memory, please use lz4frame.h instead. */ /************************************** * Version **************************************/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 6 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); @@ -70,28 +70,32 @@ int LZ4_versionNumber (void); * Simple Functions **************************************/ -int LZ4_compress (const char* source, char* dest, int sourceSize); +int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); /* -LZ4_compress() : - Compresses 'sourceSize' bytes from 'source' into 'dest'. - Destination buffer must be already allocated, - and must be sized to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() - inputSize : Max supported value is LZ4_MAX_INPUT_SIZE - return : the number of bytes written in buffer dest - or 0 if the compression fails +LZ4_compress_default() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails LZ4_decompress_safe() : - compressedSize : is obviously the source size - maxDecompressedSize : is the size of the destination buffer, which must be already allocated. - return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize) - If the destination buffer is not large enough, decoding will stop and output an error code (<0). + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, - and never writes outside of output buffer, nor reads outside of input buffer. - It is also protected against malicious data packets. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. */ @@ -99,45 +103,55 @@ LZ4_decompress_safe() : * Advanced Functions **************************************/ #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) /* LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) - This function is primarily useful for memory allocation purposes (output buffer size). + This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - - isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE - return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) + Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) */ -int LZ4_compressBound(int isize); - +int LZ4_compressBound(int inputSize); /* -LZ4_compress_limitedOutput() : - Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This saves time and memory on detecting non-compressible (or barely compressible) data. - This function never writes outside of provided output buffer. - - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxOutputSize : is the size of the destination buffer (which must be already allocated) - return : the number of bytes written in buffer 'dest' - or 0 if compression fails +LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows to select an "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "0" means "use Default value" (see lz4.c) + An acceleration value of "1" is the same as regular LZ4_compress_default() */ -int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); /* -LZ4_compress_withState() : - Same compression functions, but using an externally allocated memory space to store compression state. +LZ4_compress_fast_extState() : + Same compression function, just using an externally allocated memory space to store compression state. Use LZ4_sizeofState() to know how much memory must be allocated, - and then, provide it as 'void* state' to compression functions. + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. */ int LZ4_sizeofState(void); -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); + + +/* +LZ4_compress_destSize() : + Reverse the logic, by compressing as much data as possible from 'source' buffer + into already allocated buffer 'dest' of size 'targetDestSize'. + This function either compresses the entire 'source' content into 'dest' if it's large enough, + or fill 'dest' buffer completely with as much data as possible from 'source'. + Original idea by WiredTiger team. + *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. + New value is necessarily <= old value. + return : Nb bytes written into 'dest' (necessarily <= targetDestSize) + or 0 if compression fails +*/ +int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); /* @@ -153,7 +167,6 @@ LZ4_decompress_fast() : */ int LZ4_decompress_fast (const char* source, char* dest, int originalSize); - /* LZ4_decompress_safe_partial() : This function decompress a compressed block of size 'compressedSize' at position 'source' @@ -172,7 +185,6 @@ int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedS /*********************************************** * Streaming Compression Functions ***********************************************/ - #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) /* @@ -188,7 +200,7 @@ typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; * LZ4_resetStream * Use this function to init an allocated LZ4_stream_t structure */ -void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); +void LZ4_resetStream (LZ4_stream_t* streamPtr); /* * LZ4_createStream will allocate and initialize an LZ4_stream_t structure @@ -197,7 +209,7 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); * They are more future proof, in case of a change of LZ4_stream_t size. */ LZ4_stream_t* LZ4_createStream(void); -int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); +int LZ4_freeStream (LZ4_stream_t* streamPtr); /* * LZ4_loadDict @@ -206,32 +218,27 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); * Loading a size of 0 is allowed. * Return : dictionary size, in bytes (necessarily <= 64 KB) */ -int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize); +int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /* - * LZ4_compress_continue - * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio - * Previous data blocks are assumed to still be present at their previous location. - * dest buffer must be already allocated, and sized to at least LZ4_compressBound(inputSize) + * LZ4_compress_fast_continue + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. */ -int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); - -/* - * LZ4_compress_limitedOutput_continue - * Same as before, but also specify a maximum target compressed size (maxOutputSize) - * If objective cannot be met, compression exits, and returns a zero. - */ -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); /* * LZ4_saveDict * If previously compressed data block is not guaranteed to remain available at its memory location * save it into a safer place (char* safeBuffer) * Note : you don't need to call LZ4_loadDict() afterwards, - * dictionary is immediately usable, you can therefore call again LZ4_compress_continue() + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue() * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error */ -int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); +int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); /************************************************ @@ -266,8 +273,18 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) - If this condition is not possible, save the relevant part of decoded data into a safe buffer, - and indicate where is its new address using LZ4_setStreamDecode() + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); @@ -277,8 +294,8 @@ int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const ch Advanced decoding functions : *_usingDict() : These decoding functions work the same as - a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue() - They are stand-alone and don't use nor update an LZ4_streamDecode_t structure. + a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue() + They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure. */ int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); @@ -288,27 +305,55 @@ int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalS /************************************** * Obsolete Functions **************************************/ -/* -Obsolete decompression functions -These function names are deprecated and should no longer be used. -They are only provided here for compatibility with older user programs. -- LZ4_uncompress is the same as LZ4_decompress_fast -- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe -These function prototypes are now disabled; uncomment them if you really need them. -It is highly recommended to stop using these functions and migrate to newer ones */ +/* Deprecate Warnings */ +/* Should these warnings messages be a problem, + it is generally possible to disable them, + with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual for example. + You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ +#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ4_GCC_VERSION >= 301) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# define LZ4_DEPRECATED(message) +# endif +#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */ + +/* Obsolete compression functions */ +/* These functions are planned to start generate warnings by r131 approximately */ +int LZ4_compress (const char* source, char* dest, int sourceSize); +int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Obsolete decompression functions */ +/* These function names are completely deprecated and must no longer be used. + They are only provided here for compatibility with older programs. + - LZ4_uncompress is the same as LZ4_decompress_fast + - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe + These function prototypes are now disabled; uncomment them only if you really need them. + It is highly recommended to stop using these prototypes and migrate to maintained ones */ /* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ /* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ - /* Obsolete streaming functions; use new streaming interface whenever possible */ -void* LZ4_create (const char* inputBuffer); -int LZ4_sizeofStreamState(void); -int LZ4_resetStreamState(void* state, const char* inputBuffer); -char* LZ4_slideInputBuffer (void* state); +LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); /* Obsolete streaming decoding functions */ -int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize); -int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize); +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); #if defined (__cplusplus) diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 474b196..d1733d0 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -39,7 +39,7 @@ You can contact the author at : /************************************** -Compiler Options +* Compiler Options **************************************/ #ifdef _MSC_VER /* Visual Studio */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ @@ -100,16 +100,18 @@ typedef unsigned long long U64; #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U #define LZ4F_MAGICNUMBER 0x184D2204U #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U -#define LZ4F_MAXHEADERFRAME_SIZE 15 -#define LZ4F_BLOCKSIZEID_DEFAULT max64KB +#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB + +static const size_t minFHSize = 7; +static const size_t maxFHSize = 15; +static const size_t BHSize = 4; +static const int minHClevel = 3; -static const size_t minFHSize = 5; -static const U32 minHClevel = 3; /************************************** * Structures and local types **************************************/ -typedef struct +typedef struct LZ4F_cctx_s { LZ4F_preferences_t prefs; U32 version; @@ -123,13 +125,14 @@ typedef struct XXH32_state_t xxh; void* lz4CtxPtr; U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ -} LZ4F_cctx_internal_t; +} LZ4F_cctx_t; -typedef struct +typedef struct LZ4F_dctx_s { LZ4F_frameInfo_t frameInfo; U32 version; U32 dStage; + U64 frameRemainingSize; size_t maxBlockSize; size_t maxBufferSize; const BYTE* srcExpect; @@ -137,14 +140,14 @@ typedef struct size_t tmpInSize; size_t tmpInTarget; BYTE* tmpOutBuffer; - BYTE* dict; + const BYTE* dict; size_t dictSize; BYTE* tmpOut; size_t tmpOutSize; size_t tmpOutStart; XXH32_state_t xxh; BYTE header[16]; -} LZ4F_dctx_internal_t; +} LZ4F_dctx_t; /************************************** @@ -156,7 +159,7 @@ static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING unsigned LZ4F_isError(LZ4F_errorCode_t code) { - return (code > (LZ4F_errorCode_t)(-ERROR_maxCode)); + return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode)); } const char* LZ4F_getErrorName(LZ4F_errorCode_t code) @@ -176,7 +179,7 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID) if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; blockSizeID -= 4; - if (blockSizeID > 3) return (size_t)-ERROR_maxBlockSize_invalid; + if (blockSizeID > 3) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid; return blockSizes[blockSizeID]; } @@ -187,7 +190,7 @@ static U32 LZ4F_readLE32 (const BYTE* srcPtr) U32 value32 = srcPtr[0]; value32 += (srcPtr[1]<<8); value32 += (srcPtr[2]<<16); - value32 += (srcPtr[3]<<24); + value32 += ((U32)srcPtr[3])<<24; return value32; } @@ -204,7 +207,7 @@ static U64 LZ4F_readLE64 (const BYTE* srcPtr) U64 value64 = srcPtr[0]; value64 += (srcPtr[1]<<8); value64 += (srcPtr[2]<<16); - value64 += (srcPtr[3]<<24); + value64 += ((U64)srcPtr[3]<<24); value64 += ((U64)srcPtr[4]<<32); value64 += ((U64)srcPtr[5]<<40); value64 += ((U64)srcPtr[6]<<48); @@ -235,15 +238,15 @@ static BYTE LZ4F_headerChecksum (const void* header, size_t length) /************************************** * Simple compression functions **************************************/ -static blockSizeID_t LZ4F_optimalBSID(const blockSizeID_t requestedBSID, const size_t srcSize) +static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize) { - blockSizeID_t proposedBSID = max64KB; + LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; size_t maxBlockSize = 64 KB; while (requestedBSID > proposedBSID) { if (srcSize <= maxBlockSize) return proposedBSID; - proposedBSID = (blockSizeID_t)((int)proposedBSID + 1); + proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); maxBlockSize <<= 2; } return requestedBSID; @@ -262,7 +265,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; - headerSize = 15; /* header size, including magic number and frame content size*/ + headerSize = maxFHSize; /* header size, including magic number and frame content size*/ streamSize = LZ4F_compressBound(srcSize, &prefs); return headerSize + streamSize; @@ -280,7 +283,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere */ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { - LZ4F_cctx_internal_t cctxI; + LZ4F_cctx_t cctxI; LZ4_stream_t lz4ctx; LZ4F_preferences_t prefs; LZ4F_compressOptions_t options; @@ -295,16 +298,14 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf cctxI.version = LZ4F_VERSION; cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ - if (preferencesPtr!=NULL) prefs = *preferencesPtr; + if (preferencesPtr!=NULL) + prefs = *preferencesPtr; else - { memset(&prefs, 0, sizeof(prefs)); - prefs.frameInfo.contentSize = (U64)srcSize; - } if (prefs.frameInfo.contentSize != 0) - prefs.frameInfo.contentSize = (U64)srcSize; /* correct content size if selected (!=0) */ + prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ - if (prefs.compressionLevel < minHClevel) + if (prefs.compressionLevel < (int)minHClevel) { cctxI.lz4CtxPtr = &lz4ctx; cctxI.lz4CtxLevel = 1; @@ -313,12 +314,12 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); prefs.autoFlush = 1; if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) - prefs.frameInfo.blockMode = blockIndependent; /* no need for linked blocks */ + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */ options.stableSrc = 1; if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) - return (size_t)-ERROR_dstMaxSize_tooSmall; + return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ if (LZ4F_isError(errorCode)) return errorCode; @@ -332,7 +333,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf if (LZ4F_isError(errorCode)) return errorCode; dstPtr += errorCode; - if (prefs.compressionLevel >= minHClevel) /* no allocation necessary with lz4 fast */ + if (prefs.compressionLevel >= (int)minHClevel) /* no allocation necessary with lz4 fast */ FREEMEM(cctxI.lz4CtxPtr); return (dstPtr - dstStart); @@ -340,8 +341,8 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf /*********************************** -* Advanced compression functions -* *********************************/ +* Advanced compression functions +***********************************/ /* LZ4F_createCompressionContext() : * The first thing to do is to create a compressionContext object, which will be used in all compression operations. @@ -353,29 +354,32 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf */ LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) { - LZ4F_cctx_internal_t* cctxPtr; + LZ4F_cctx_t* cctxPtr; - cctxPtr = (LZ4F_cctx_internal_t*)ALLOCATOR(sizeof(LZ4F_cctx_internal_t)); - if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-ERROR_allocation_failed); + cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t)); + if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-LZ4F_ERROR_allocation_failed); cctxPtr->version = version; cctxPtr->cStage = 0; /* Next stage : write header */ *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; - return OK_NoError; + return LZ4F_OK_NoError; } LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) { - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)LZ4F_compressionContext; + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; - FREEMEM(cctxPtr->lz4CtxPtr); - FREEMEM(cctxPtr->tmpBuff); - FREEMEM(LZ4F_compressionContext); + if (cctxPtr != NULL) /* null pointers can be safely provided to this function, like free() */ + { + FREEMEM(cctxPtr->lz4CtxPtr); + FREEMEM(cctxPtr->tmpBuff); + FREEMEM(LZ4F_compressionContext); + } - return OK_NoError; + return LZ4F_OK_NoError; } @@ -388,25 +392,25 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefNull; - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; BYTE* headerStart; size_t requiredBuffSize; - if (dstMaxSize < LZ4F_MAXHEADERFRAME_SIZE) return (size_t)-ERROR_dstMaxSize_tooSmall; - if (cctxPtr->cStage != 0) return (size_t)-ERROR_GENERIC; + if (dstMaxSize < maxFHSize) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 0) return (size_t)-LZ4F_ERROR_GENERIC; memset(&prefNull, 0, sizeof(prefNull)); if (preferencesPtr == NULL) preferencesPtr = &prefNull; cctxPtr->prefs = *preferencesPtr; /* ctx Management */ { - U32 tableID = cctxPtr->prefs.compressionLevel<minHClevel ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + U32 tableID = (cctxPtr->prefs.compressionLevel < minHClevel) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ if (cctxPtr->lz4CtxLevel < tableID) { FREEMEM(cctxPtr->lz4CtxPtr); - if (cctxPtr->prefs.compressionLevel<minHClevel) + if (cctxPtr->prefs.compressionLevel < minHClevel) cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); else cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); @@ -418,16 +422,16 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); - requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 128 KB); + requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); if (preferencesPtr->autoFlush) - requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == blockLinked) * 64 KB; /* just needs dict */ + requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */ if (cctxPtr->maxBufferSize < requiredBuffSize) { cctxPtr->maxBufferSize = requiredBuffSize; FREEMEM(cctxPtr->tmpBuff); cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); - if (cctxPtr->tmpBuff == NULL) return (size_t)-ERROR_allocation_failed; + if (cctxPtr->tmpBuff == NULL) return (size_t)-LZ4F_ERROR_allocation_failed; } cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpInSize = 0; @@ -443,10 +447,10 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds headerStart = dstPtr; /* FLG Byte */ - *dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ - + (BYTE)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ - + (BYTE)((cctxPtr->prefs.frameInfo.contentSize > 0) << 3); /* Frame content size */ + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ + + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ /* BD Byte */ *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); /* Optional Frame content size field */ @@ -468,15 +472,16 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds /* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. * The LZ4F_frameInfo_t structure is optional : -* you can provide NULL as argument, all preferences will then be set to default. +* you can provide NULL as argument, preferences will then be set to cover worst case situations. * */ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) { LZ4F_preferences_t prefsNull; memset(&prefsNull, 0, sizeof(prefsNull)); + prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ { const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; - blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + LZ4F_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; size_t blockSize = LZ4F_getBlockSize(bid); unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; @@ -522,21 +527,21 @@ static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) { (void) level; - return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); } -static compressFunc_t LZ4F_selectCompression(blockMode_t blockMode, U32 level) +static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) { if (level < minHClevel) { - if (blockMode == blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; + if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; return LZ4F_localLZ4_compress_limitedOutput_continue; } - if (blockMode == blockIndependent) return LZ4_compressHC2_limitedOutput_withStateHC; + if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC; return LZ4F_localLZ4_compressHC_limitedOutput_continue; } -static int LZ4F_localSaveDict(LZ4F_cctx_internal_t* cctxPtr) +static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) { if (cctxPtr->prefs.compressionLevel < minHClevel) return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); @@ -557,7 +562,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) { LZ4F_compressOptions_t cOptionsNull; - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; size_t blockSize = cctxPtr->maxBlockSize; const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcPtr + srcSize; @@ -567,8 +572,8 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d compressFunc_t compress; - if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; - if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC; + if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; memset(&cOptionsNull, 0, sizeof(cOptionsNull)); if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; @@ -596,7 +601,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += blockSize; + if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; cctxPtr->tmpInSize = 0; } } @@ -618,7 +623,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d } /* preserve dictionary if necessary */ - if ((cctxPtr->prefs.frameInfo.blockMode==blockLinked) && (lastBlockCompressed==fromSrcBuffer)) + if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { if (compressOptionsPtr->stableSrc) { @@ -627,13 +632,13 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d else { int realDictSize = LZ4F_localSaveDict(cctxPtr); - if (realDictSize==0) return (size_t)-ERROR_GENERIC; + if (realDictSize==0) return (size_t)-LZ4F_ERROR_GENERIC; cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; } } /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily blockLinked && lastBlockCompressed==fromTmpBuffer */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */ && !(cctxPtr->prefs.autoFlush)) { int realDictSize = LZ4F_localSaveDict(cctxPtr); @@ -649,7 +654,7 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d cctxPtr->tmpInSize = sizeToCopy; } - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); cctxPtr->totalInSize += srcSize; @@ -667,15 +672,15 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d */ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; compressFunc_t compress; if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ - if (cctxPtr->cStage != 1) return (size_t)-ERROR_GENERIC; - if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ + if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC; + if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ (void)compressOptionsPtr; /* not yet useful */ /* select compression function */ @@ -683,11 +688,11 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, /* compress tmp buffer */ dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); - if (cctxPtr->prefs.frameInfo.blockMode==blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; cctxPtr->tmpInSize = 0; /* keep tmpIn within limits */ - if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily blockLinked */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily LZ4F_blockLinked */ { int realDictSize = LZ4F_localSaveDict(cctxPtr); cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; @@ -708,7 +713,7 @@ size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, */ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) { - LZ4F_cctx_internal_t* cctxPtr = (LZ4F_cctx_internal_t*)compressionContext; + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; BYTE* const dstStart = (BYTE*)dstBuffer; BYTE* dstPtr = dstStart; size_t errorCode; @@ -720,7 +725,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB LZ4F_writeLE32(dstPtr, 0); dstPtr+=4; /* endMark */ - if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled) + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { U32 xxh = XXH32_digest(&(cctxPtr->xxh)); LZ4F_writeLE32(dstPtr, xxh); @@ -732,7 +737,7 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB if (cctxPtr->prefs.frameInfo.contentSize) { if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) - return (size_t)-ERROR_frameSize_wrong; + return (size_t)-LZ4F_ERROR_frameSize_wrong; } return dstPtr - dstStart; @@ -754,23 +759,28 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB */ LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) { - LZ4F_dctx_internal_t* dctxPtr; + LZ4F_dctx_t* dctxPtr; - dctxPtr = (LZ4F_dctx_internal_t*)ALLOCATOR(sizeof(LZ4F_dctx_internal_t)); - if (dctxPtr==NULL) return (LZ4F_errorCode_t)-ERROR_GENERIC; + dctxPtr = (LZ4F_dctx_t*)ALLOCATOR(sizeof(LZ4F_dctx_t)); + if (dctxPtr==NULL) return (LZ4F_errorCode_t)-LZ4F_ERROR_GENERIC; dctxPtr->version = versionNumber; - *LZ4F_decompressionContextPtr = (LZ4F_compressionContext_t)dctxPtr; - return OK_NoError; + *LZ4F_decompressionContextPtr = (LZ4F_decompressionContext_t)dctxPtr; + return LZ4F_OK_NoError; } LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_decompressionContext) { - LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)LZ4F_decompressionContext; - FREEMEM(dctxPtr->tmpIn); - FREEMEM(dctxPtr->tmpOutBuffer); - FREEMEM(dctxPtr); - return OK_NoError; + LZ4F_errorCode_t result = LZ4F_OK_NoError; + LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)LZ4F_decompressionContext; + if (dctxPtr != NULL) /* can accept NULL input, like free() */ + { + result = (LZ4F_errorCode_t)dctxPtr->dStage; + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + FREEMEM(dctxPtr); + } + return result; } @@ -781,8 +791,9 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_ typedef enum { dstage_getHeader=0, dstage_storeHeader, dstage_getCBlockSize, dstage_storeCBlockSize, dstage_copyDirect, - dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock, - dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getCBlock, dstage_storeCBlock, + dstage_decodeCBlock, dstage_decodeCBlock_intoDst, + dstage_decodeCBlock_intoTmp, dstage_flushOut, dstage_getSuffix, dstage_storeSuffix, dstage_getSFrameSize, dstage_storeSFrameSize, dstage_skipSkippable @@ -794,8 +805,9 @@ typedef enum { dstage_getHeader=0, dstage_storeHeader, or an error code (testable with LZ4F_isError()) output : set internal values of dctx, such as dctxPtr->frameInfo and dctxPtr->dStage. + input : srcVoidPtr points at the **beginning of the frame** */ -static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) +static size_t LZ4F_decodeHeader(LZ4F_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) { BYTE FLG, BD, HC; unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; @@ -804,13 +816,13 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo const BYTE* srcPtr = (const BYTE*)srcVoidPtr; /* need to decode header to get frameInfo */ - if (srcSize < minFHSize) return (size_t)-ERROR_GENERIC; /* minimal header size */ + if (srcSize < minFHSize) return (size_t)-LZ4F_ERROR_frameHeader_incomplete; /* minimal frame header size */ memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); - /* skippable frames */ + /* special case : skippable frames */ if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { - dctxPtr->frameInfo.frameType = skippableFrame; + dctxPtr->frameInfo.frameType = LZ4F_skippableFrame; if (srcVoidPtr == (void*)(dctxPtr->header)) { dctxPtr->tmpInSize = srcSize; @@ -826,7 +838,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo } /* control magic number */ - if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_frameType_unknown; + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-LZ4F_ERROR_frameType_unknown; dctxPtr->frameInfo.frameType = LZ4F_frame; /* Flags */ @@ -838,10 +850,11 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo contentChecksumFlag = (FLG>>2) & _1BIT; /* Frame Header Size */ - frameHeaderSize = contentSizeFlag ? 15 : 7; + frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; if (srcSize < frameHeaderSize) { + /* not enough input to fully decode frame header */ if (srcPtr != dctxPtr->header) memcpy(dctxPtr->header, srcPtr, srcSize); dctxPtr->tmpInSize = srcSize; @@ -854,39 +867,39 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo blockSizeID = (BD>>4) & _3BITS; /* validate */ - if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */ - if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */ - if (((FLG>>0)&_2BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ - if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */ - if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */ - if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */ + if (version != 1) return (size_t)-LZ4F_ERROR_headerVersion_wrong; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-LZ4F_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ + if (((FLG>>0)&_2BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */ + if (((BD>>7)&_1BIT) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bit */ + if (blockSizeID < 4) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid; /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */ /* check */ HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); - if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */ + if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LZ4F_ERROR_headerChecksum_invalid; /* Bad header checksum error */ /* save */ - dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode; - dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag; - dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID; + dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); if (contentSizeFlag) - dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); /* init */ if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); /* alloc */ - bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==blockLinked) * 128 KB); + bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); if (bufferNeeded > dctxPtr->maxBufferSize) /* tmp buffers too small */ { FREEMEM(dctxPtr->tmpIn); FREEMEM(dctxPtr->tmpOutBuffer); dctxPtr->maxBufferSize = bufferNeeded; dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); - if (dctxPtr->tmpIn == NULL) return (size_t)-ERROR_GENERIC; + if (dctxPtr->tmpIn == NULL) return (size_t)-LZ4F_ERROR_GENERIC; dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize); - if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-ERROR_GENERIC; + if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LZ4F_ERROR_GENERIC; } dctxPtr->tmpInSize = 0; dctxPtr->tmpInTarget = 0; @@ -912,28 +925,32 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, * or an error code which can be tested using LZ4F_isError(). */ -LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionContext, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr) +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dCtx, LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) { - LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; + LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)dCtx; - if (dctxPtr->dStage == dstage_getHeader) + if (dctxPtr->dStage > dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ { - size_t frameHeaderSize = LZ4F_decodeHeader(dctxPtr, srcBuffer, *srcSizePtr); - if (LZ4F_isError(frameHeaderSize)) return frameHeaderSize; - *srcSizePtr = frameHeaderSize; /* nb Bytes consumed */ - *frameInfoPtr = dctxPtr->frameInfo; /* copy into */ - dctxPtr->srcExpect = NULL; - return 4; /* nextSrcSizeHint : 4 == block header size */ + size_t o=0, i=0; + /* frameInfo already decoded */ + *srcSizePtr = 0; + *frameInfoPtr = dctxPtr->frameInfo; + return LZ4F_decompress(dCtx, NULL, &o, NULL, &i, NULL); + } + else + { + size_t o=0; + size_t nextSrcSize = LZ4F_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); + if (dctxPtr->dStage <= dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ + return (size_t)-LZ4F_ERROR_frameHeader_incomplete; + *frameInfoPtr = dctxPtr->frameInfo; + return nextSrcSize; } - - /* frameInfo already decoded */ - *srcSizePtr = 0; - *frameInfoPtr = dctxPtr->frameInfo; - return 0; } -/* redirector, with common prototype */ +/* trivial redirector, for common prototype */ static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) { (void)dictStart; (void)dictSize; @@ -941,10 +958,10 @@ static int LZ4F_decompress_safe (const char* source, char* dest, int compressedS } -static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +static void LZ4F_updateDict(LZ4F_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) { if (dctxPtr->dictSize==0) - dctxPtr->dict = (BYTE*)dstPtr; /* priority to dictionary continuity */ + dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) /* dictionary continuity */ { @@ -954,7 +971,7 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s if (dstPtr - dstPtr0 + dstSize >= 64 KB) /* dstBuffer large enough to become dictionary */ { - dctxPtr->dict = (BYTE*)dstPtr0; + dctxPtr->dict = (const BYTE*)dstPtr0; dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; return; } @@ -970,7 +987,7 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s { size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; size_t copySize = 64 KB - dctxPtr->tmpOutSize; - BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; @@ -986,10 +1003,10 @@ static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, s if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) /* tmp buffer not large enough */ { size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ - memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); dctxPtr->dictSize = preserveSize; } - memcpy(dctxPtr->dict + dctxPtr->dictSize, dstPtr, dstSize); + memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); dctxPtr->dictSize += dstSize; return; } @@ -1029,7 +1046,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, const void* srcBuffer, size_t* srcSizePtr, const LZ4F_decompressOptions_t* decompressOptionsPtr) { - LZ4F_dctx_internal_t* dctxPtr = (LZ4F_dctx_internal_t*)decompressionContext; + LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)decompressionContext; LZ4F_decompressOptions_t optionsNull; const BYTE* const srcStart = (const BYTE*)srcBuffer; const BYTE* const srcEnd = srcStart + *srcSizePtr; @@ -1050,7 +1067,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, /* expect to continue decoding src buffer where it left previously */ if (dctxPtr->srcExpect != NULL) { - if (srcStart != dctxPtr->srcExpect) return (size_t)-ERROR_wrongSrcPtr; + if (srcStart != dctxPtr->srcExpect) return (size_t)-LZ4F_ERROR_srcPtr_wrong; } /* programmed as a state machine */ @@ -1063,7 +1080,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, case dstage_getHeader: { - if (srcEnd-srcPtr >= 7) + if ((size_t)(srcEnd-srcPtr) >= maxFHSize) /* enough to decode - shortcut */ { LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); if (LZ4F_isError(errorCode)) return errorCode; @@ -1071,7 +1088,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } dctxPtr->tmpInSize = 0; - dctxPtr->tmpInTarget = 7; + dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ dctxPtr->dStage = dstage_storeHeader; } @@ -1084,7 +1101,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, srcPtr += sizeToCopy; if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) { - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ doAnotherStage = 0; /* not enough src data, ask for some more */ break; } @@ -1097,10 +1114,10 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, case dstage_getCBlockSize: { - if ((srcEnd - srcPtr) >= 4) + if ((size_t)(srcEnd - srcPtr) >= BHSize) { selectedIn = srcPtr; - srcPtr += 4; + srcPtr += BHSize; } else { @@ -1113,15 +1130,15 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, if (dctxPtr->dStage == dstage_storeCBlockSize) case dstage_storeCBlockSize: { - size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); srcPtr += sizeToCopy; dctxPtr->tmpInSize += sizeToCopy; - if (dctxPtr->tmpInSize < 4) /* not enough input to get full cBlockSize; wait for more */ + if (dctxPtr->tmpInSize < BHSize) /* not enough input to get full cBlockSize; wait for more */ { - nextSrcSizeHint = 4 - dctxPtr->tmpInSize; - doAnotherStage=0; + nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; + doAnotherStage = 0; break; } selectedIn = dctxPtr->tmpIn; @@ -1135,7 +1152,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, dctxPtr->dStage = dstage_getSuffix; break; } - if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-ERROR_GENERIC; /* invalid cBlockSize */ + if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LZ4F_ERROR_GENERIC; /* invalid cBlockSize */ dctxPtr->tmpInTarget = nextCBlockSize; if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) { @@ -1145,7 +1162,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, dctxPtr->dStage = dstage_getCBlock; if (dstPtr==dstEnd) { - nextSrcSizeHint = nextCBlockSize + 4; + nextSrcSizeHint = nextCBlockSize + BHSize; doAnotherStage = 0; } break; @@ -1158,10 +1175,10 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; memcpy(dstPtr, srcPtr, sizeToCopy); if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameInfo.contentSize -= sizeToCopy; + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) + if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); srcPtr += sizeToCopy; @@ -1172,7 +1189,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, break; } dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ - nextSrcSizeHint = dctxPtr->tmpInTarget + 4; + nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; doAnotherStage = 0; break; } @@ -1200,7 +1217,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, srcPtr += sizeToCopy; if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ { - nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4; + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; doAnotherStage=0; break; } @@ -1223,18 +1240,18 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, int (*decoder)(const char*, char*, int, int, const char*, int); int decodedSize; - if (dctxPtr->frameInfo.blockMode == blockLinked) + if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) decoder = LZ4_decompress_safe_usingDict; else decoder = LZ4F_decompress_safe; decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-ERROR_GENERIC; /* decompression failed */ + if (decodedSize < 0) return (size_t)-LZ4F_ERROR_GENERIC; /* decompression failed */ if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameInfo.contentSize -= decodedSize; + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) + if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); dstPtr += decodedSize; @@ -1248,22 +1265,22 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, int (*decoder)(const char*, char*, int, int, const char*, int); int decodedSize; - if (dctxPtr->frameInfo.blockMode == blockLinked) + if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) decoder = LZ4_decompress_safe_usingDict; else decoder = LZ4F_decompress_safe; /* ensure enough place for tmpOut */ - if (dctxPtr->frameInfo.blockMode == blockLinked) + if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) { if (dctxPtr->dict == dctxPtr->tmpOutBuffer) { if (dctxPtr->dictSize > 128 KB) { - memcpy(dctxPtr->dict, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); dctxPtr->dictSize = 64 KB; } - dctxPtr->tmpOut = dctxPtr->dict + dctxPtr->dictSize; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; } else /* dict not within tmp */ { @@ -1275,9 +1292,9 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, /* Decode */ decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); - if (decodedSize < 0) return (size_t)-ERROR_decompressionFailed; /* decompression failed */ + if (decodedSize < 0) return (size_t)-LZ4F_ERROR_decompressionFailed; /* decompression failed */ if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); - if (dctxPtr->frameInfo.contentSize) dctxPtr->frameInfo.contentSize -= decodedSize; + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; dctxPtr->tmpOutSize = decodedSize; dctxPtr->tmpOutStart = 0; dctxPtr->dStage = dstage_flushOut; @@ -1291,7 +1308,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); /* dictionary management */ - if (dctxPtr->frameInfo.blockMode==blockLinked) + if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); dctxPtr->tmpOutStart += sizeToCopy; @@ -1303,7 +1320,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, dctxPtr->dStage = dstage_getCBlockSize; break; } - nextSrcSizeHint = 4; + nextSrcSizeHint = BHSize; doAnotherStage = 0; /* still some data to flush */ break; } @@ -1311,7 +1328,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, case dstage_getSuffix: { size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; - if (dctxPtr->frameInfo.contentSize) return (size_t)-ERROR_frameSize_wrong; /* incorrect frame size decoded */ + if (dctxPtr->frameRemainingSize) return (size_t)-LZ4F_ERROR_frameSize_wrong; /* incorrect frame size decoded */ if (suffixSize == 0) /* frame completed */ { nextSrcSizeHint = 0; @@ -1352,7 +1369,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, { U32 readCRC = LZ4F_readLE32(selectedIn); U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); - if (readCRC != resultCRC) return (size_t)-ERROR_checksum_invalid; + if (readCRC != resultCRC) return (size_t)-LZ4F_ERROR_contentChecksum_invalid; nextSrcSizeHint = 0; dctxPtr->dStage = dstage_getHeader; doAnotherStage = 0; @@ -1392,7 +1409,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, selectedIn = dctxPtr->header + 4; } - /* case dstage_decodeSBlockSize: */ /* no direct access */ + /* case dstage_decodeSFrameSize: */ /* no direct access */ { size_t SFrameSize = LZ4F_readLE32(selectedIn); dctxPtr->frameInfo.contentSize = SFrameSize; @@ -1417,7 +1434,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, } /* preserve dictionary within tmp if necessary */ - if ( (dctxPtr->frameInfo.blockMode==blockLinked) + if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) &&(!decompressOptionsPtr->stableDst) &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) @@ -1427,7 +1444,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, { size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; size_t copySize = 64 KB - dctxPtr->tmpOutSize; - BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; if (copySize > preserveSize) copySize = preserveSize; @@ -1439,7 +1456,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, else { size_t newDictSize = dctxPtr->dictSize; - BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; if ((newDictSize) > 64 KB) newDictSize = 64 KB; memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); diff --git a/lib/lz4frame.h b/lib/lz4frame.h index 9d12c7d..05fbc5f 100644 --- a/lib/lz4frame.h +++ b/lib/lz4frame.h @@ -33,7 +33,7 @@ */ /* LZ4F is a stand-alone API to create LZ4-compressed frames - * fully conformant to specification v1.4.1. + * fully conformant to specification v1.5.1. * All related operations, including memory management, are handled by the library. * You don't need lz4.h when using lz4frame.h. * */ @@ -45,7 +45,7 @@ extern "C" { #endif /************************************** - Includes +* Includes **************************************/ #include <stddef.h> /* size_t */ @@ -62,25 +62,66 @@ const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code str /************************************** * Frame compression types * ************************************/ -typedef enum { LZ4F_default=0, max64KB=4, max256KB=5, max1MB=6, max4MB=7 } blockSizeID_t; -typedef enum { blockLinked=0, blockIndependent} blockMode_t; -typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t; -typedef enum { LZ4F_frame=0, skippableFrame } frameType_t; +//#define LZ4F_DISABLE_OBSOLETE_ENUMS +#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +# define LZ4F_OBSOLETE_ENUM(x) ,x +#else +# define LZ4F_OBSOLETE_ENUM(x) +#endif + +typedef enum { + LZ4F_default=0, + LZ4F_max64KB=4, + LZ4F_max256KB=5, + LZ4F_max1MB=6, + LZ4F_max4MB=7 + LZ4F_OBSOLETE_ENUM(max64KB = LZ4F_max64KB) + LZ4F_OBSOLETE_ENUM(max256KB = LZ4F_max256KB) + LZ4F_OBSOLETE_ENUM(max1MB = LZ4F_max1MB) + LZ4F_OBSOLETE_ENUM(max4MB = LZ4F_max4MB) +} LZ4F_blockSizeID_t; + +typedef enum { + LZ4F_blockLinked=0, + LZ4F_blockIndependent + LZ4F_OBSOLETE_ENUM(blockLinked = LZ4F_blockLinked) + LZ4F_OBSOLETE_ENUM(blockIndependent = LZ4F_blockIndependent) +} LZ4F_blockMode_t; + +typedef enum { + LZ4F_noContentChecksum=0, + LZ4F_contentChecksumEnabled + LZ4F_OBSOLETE_ENUM(noContentChecksum = LZ4F_noContentChecksum) + LZ4F_OBSOLETE_ENUM(contentChecksumEnabled = LZ4F_contentChecksumEnabled) +} LZ4F_contentChecksum_t; + +typedef enum { + LZ4F_frame=0, + LZ4F_skippableFrame + LZ4F_OBSOLETE_ENUM(skippableFrame = LZ4F_skippableFrame) +} LZ4F_frameType_t; + +#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +typedef LZ4F_blockSizeID_t blockSizeID_t; +typedef LZ4F_blockMode_t blockMode_t; +typedef LZ4F_frameType_t frameType_t; +typedef LZ4F_contentChecksum_t contentChecksum_t; +#endif typedef struct { - blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ - blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ - contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ - frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ - unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ - unsigned reserved[2]; /* must be zero for forward compatibility */ + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ + unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ + unsigned reserved[2]; /* must be zero for forward compatibility */ } LZ4F_frameInfo_t; typedef struct { LZ4F_frameInfo_t frameInfo; - unsigned compressionLevel; /* 0 == default (fast mode); values above 16 count as 16 */ - unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ - unsigned reserved[4]; /* must be zero for forward compatibility */ + int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */ + unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ + unsigned reserved[4]; /* must be zero for forward compatibility */ } LZ4F_preferences_t; @@ -91,7 +132,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); /* LZ4F_compressFrame() - * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5 + * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1 * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) @@ -103,9 +144,9 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf /********************************** - * Advanced compression functions - * ********************************/ -typedef void* LZ4F_compressionContext_t; +* Advanced compression functions +**********************************/ +typedef struct LZ4F_cctx_s* LZ4F_compressionContext_t; /* must be aligned on 8-bytes */ typedef struct { unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */ @@ -141,16 +182,18 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t cctx, void* dstBuffer, size_ size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); /* LZ4F_compressBound() : * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. - * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to default. - * Note that different preferences will produce in different results. + * Different preferences can produce different results. + * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. + * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) */ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); /* LZ4F_compressUpdate() * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. - * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) - * You can get the minimum value of dstMaxSize by using LZ4F_compressBound() + * You can get the minimum value of dstMaxSize by using LZ4F_compressBound(). + * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). + * LZ4F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs. * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. * The function outputs an error code if it fails (can be tested using LZ4F_isError()) @@ -172,18 +215,18 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). * It will flush whatever data remained within compressionContext (like LZ4_flush()) * but also properly finalize the frame, with an endMark and a checksum. - * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * The function outputs an error code if it fails (can be tested using LZ4F_isError()) * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. - * A successful call to LZ4F_compressEnd() makes cctx available again for future compression work. + * A successful call to LZ4F_compressEnd() makes cctx available again for next compression task. */ /*********************************** - * Decompression functions - * *********************************/ +* Decompression functions +***********************************/ -typedef void* LZ4F_decompressionContext_t; +typedef struct LZ4F_dctx_s* LZ4F_decompressionContext_t; /* must be aligned on 8-bytes */ typedef struct { unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ @@ -202,6 +245,8 @@ LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t dctx) * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. * The result is an errorCode, which can be tested using LZ4F_isError(). * dctx memory can be released using LZ4F_freeDecompressionContext(); + * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. + * That is, it should be == 0 if decompression has been completed fully and correctly. */ @@ -211,18 +256,16 @@ size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dctx, LZ4F_frameInfo_t* frameInfoPtr, const void* srcBuffer, size_t* srcSizePtr); /* LZ4F_getFrameInfo() - * This function decodes frame header information, such as blockSize. - * It is optional : you could start by calling directly LZ4F_decompress() instead. - * The objective is to extract header information without starting decompression, typically for allocation purposes. - * The function will work only if srcBuffer points at the beginning of the frame, - * and *srcSizePtr is large enough to decode the whole header (typically, between 7 & 15 bytes). - * The result is copied into an LZ4F_frameInfo_t structure, which is pointed by frameInfoPtr, and must be already allocated. - * LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. + * This function decodes frame header information (such as max blockSize, frame checksum, etc.). + * Its usage is optional : you can start by calling directly LZ4F_decompress() instead. + * The objective is to extract frame header information, typically for allocation purposes. + * LZ4F_getFrameInfo() can also be used anytime *after* starting decompression, on any valid LZ4F_decompressionContext_t. + * The result is *copied* into an existing LZ4F_frameInfo_t structure which must be already allocated. * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). - * It is basically the frame header size. - * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call, - * or an error code which can be tested using LZ4F_isError(). + * or an error code which can be tested using LZ4F_isError() + * (typically, when there is not enough src bytes to fully decode the frame header) + * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) */ size_t LZ4F_decompress(LZ4F_decompressionContext_t dctx, diff --git a/lib/lz4frame_static.h b/lib/lz4frame_static.h index 2e56400..0d90975 100644 --- a/lib/lz4frame_static.h +++ b/lib/lz4frame_static.h @@ -40,35 +40,42 @@ extern "C" { #endif /* lz4frame_static.h should be used solely in the context of static linking. + * It contains definitions which may still change overtime. + * Never use it in the context of DLL linking. * */ /************************************** +* Includes +**************************************/ +#include "lz4frame.h" + + +/************************************** * Error management * ************************************/ #define LZ4F_LIST_ERRORS(ITEM) \ ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \ ITEM(ERROR_allocation_failed) \ ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ - ITEM(ERROR_frameSize_wrong) \ - ITEM(ERROR_frameType_unknown) \ - ITEM(ERROR_wrongSrcPtr) \ + ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcPtr_wrong) \ ITEM(ERROR_decompressionFailed) \ - ITEM(ERROR_checksum_invalid) \ + ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \ ITEM(ERROR_maxCode) -#define LZ4F_GENERATE_ENUM(ENUM) ENUM, +//#define LZ4F_DISABLE_OLD_ENUMS +#ifndef LZ4F_DISABLE_OLD_ENUMS +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM, +#else +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, +#endif typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ -/************************************** - Includes -**************************************/ -#include "lz4frame.h" - - #if defined (__cplusplus) } #endif diff --git a/lib/lz4hc.c b/lib/lz4hc.c index c7a94a0..a22ab2b 100644 --- a/lib/lz4hc.c +++ b/lib/lz4hc.c @@ -1,53 +1,53 @@ /* -LZ4 HC - High Compression Mode of LZ4 -Copyright (C) 2011-2015, Yann Collet. - -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : - - LZ4 source repository : https://github.com/Cyan4973/lz4 - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /************************************** - Tuning Parameter +* Tuning Parameter **************************************/ static const int LZ4HC_compressionLevel_default = 9; /************************************** - Includes +* Includes **************************************/ #include "lz4hc.h" /************************************** - Local Compiler Options +* Local Compiler Options **************************************/ #if defined(__GNUC__) # pragma GCC diagnostic ignored "-Wunused-function" @@ -59,18 +59,18 @@ static const int LZ4HC_compressionLevel_default = 9; /************************************** - Common LZ4 definition +* Common LZ4 definition **************************************/ #define LZ4_COMMONDEFS_ONLY #include "lz4.c" /************************************** - Local Constants +* Local Constants **************************************/ #define DICTIONARY_LOGSIZE 16 #define MAXD (1<<DICTIONARY_LOGSIZE) -#define MAXD_MASK ((U32)(MAXD - 1)) +#define MAXD_MASK (MAXD - 1) #define HASH_LOG (DICTIONARY_LOGSIZE-1) #define HASHTABLESIZE (1 << HASH_LOG) @@ -82,36 +82,36 @@ static const int g_maxCompressionLevel = 16; /************************************** - Local Types +* Local Types **************************************/ typedef struct { - U32 hashTable[HASHTABLESIZE]; + U32 hashTable[HASHTABLESIZE]; U16 chainTable[MAXD]; const BYTE* end; /* next block here to continue on current prefix */ const BYTE* base; /* All index relative to this position */ const BYTE* dictBase; /* alternate base for extDict */ - const BYTE* inputBuffer;/* deprecated */ + BYTE* inputBuffer; /* deprecated */ U32 dictLimit; /* below that point, need extDict */ U32 lowLimit; /* below that point, no more dict */ - U32 nextToUpdate; + U32 nextToUpdate; /* index from which to continue dictionary update */ U32 compressionLevel; } LZ4HC_Data_Structure; /************************************** - Local Macros +* Local Macros **************************************/ #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) -#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] -#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) +//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */ +#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } /************************************** - HC Compression +* HC Compression **************************************/ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) { @@ -119,7 +119,6 @@ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); hc4->nextToUpdate = 64 KB; hc4->base = start - 64 KB; - hc4->inputBuffer = start; hc4->end = start; hc4->dictBase = start - 64 KB; hc4->dictLimit = 64 KB; @@ -141,7 +140,7 @@ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) U32 h = LZ4HC_hashPtr(base+idx); size_t delta = idx - HashTable[h]; if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - chainTable[idx & 0xFFFF] = (U16)delta; + DELTANEXTU16(idx) = (U16)delta; HashTable[h] = idx; idx++; } @@ -197,7 +196,7 @@ FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* I if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ } } - matchIndex -= chainTable[matchIndex & 0xFFFF]; + matchIndex -= DELTANEXTU16(matchIndex); } return (int)ml; @@ -274,7 +273,7 @@ FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } } - matchIndex -= chainTable[matchIndex & 0xFFFF]; + matchIndex -= DELTANEXTU16(matchIndex); } return longest; @@ -536,63 +535,32 @@ _Search3: } -int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel) -{ - LZ4HC_Data_Structure ctx; - LZ4HC_init(&ctx, (const BYTE*)source); - return LZ4HC_compress_generic (&ctx, source, dest, inputSize, 0, compressionLevel, noLimit); -} - -int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); } - -int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) -{ - LZ4HC_Data_Structure ctx; - LZ4HC_init(&ctx, (const BYTE*)source); - return LZ4HC_compress_generic (&ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); -} - -int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compressHC2_limitedOutput(source, dest, inputSize, maxOutputSize, 0); -} - - -/***************************** - * Using external allocation - * ***************************/ int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); } - -int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel) +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source); - return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit); + LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src); + if (maxDstSize < LZ4_compressBound(srcSize)) + return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); + else + return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); } -int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize) -{ return LZ4_compressHC2_withStateHC (state, source, dest, inputSize, 0); } - - -int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) { - if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ - LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source); - return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); + LZ4HC_Data_Structure state; + return LZ4_compress_HC_extStateHC(&state, src, dst, srcSize, maxDstSize, compressionLevel); } -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) -{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); } - /************************************** - * Streaming Functions - * ************************************/ +* Streaming Functions +**************************************/ /* allocation */ LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; } +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; } /* initialization */ @@ -651,14 +619,15 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, } /* Check if blocks follow each other */ - if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); + if ((const BYTE*)source != ctxPtr->end) + LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; - if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd)) + if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { if (sourceEnd > dictEnd) sourceEnd = dictEnd; ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); @@ -669,14 +638,12 @@ static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); } -int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) +int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit); -} - -int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + if (maxOutputSize < LZ4_compressBound(inputSize)) + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + else + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); } @@ -689,7 +656,7 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; - memcpy(safeBuffer, streamPtr->end - dictSize, dictSize); + memmove(safeBuffer, streamPtr->end - dictSize, dictSize); { U32 endIndex = (U32)(streamPtr->end - streamPtr->base); streamPtr->end = (const BYTE*)safeBuffer + dictSize; @@ -703,21 +670,39 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS /*********************************** - * Deprecated Functions - ***********************************/ +* Deprecated Functions +***********************************/ +/* Deprecated compression functions */ +/* These functions are planned to start generate warnings by r131 approximately */ +int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } + + +/* Deprecated streaming functions */ +/* These functions currently generate deprecation warnings */ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) +int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); + ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer; return 0; } -void* LZ4_createHC (const char* inputBuffer) +void* LZ4_createHC (char* inputBuffer) { void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); + ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer; return hc4; } @@ -727,17 +712,6 @@ int LZ4_freeHC (void* LZ4HC_Data) return (0); } -/* -int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) -{ -return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); -} -int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) -{ -return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); -} -*/ - int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) { return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); diff --git a/lib/lz4hc.h b/lib/lz4hc.h index 4a05845..431f7c8 100644 --- a/lib/lz4hc.h +++ b/lib/lz4hc.h @@ -38,141 +38,150 @@ extern "C" { #endif +/***************************** +* Includes +*****************************/ +#include <stddef.h> /* size_t */ -int LZ4_compressHC (const char* source, char* dest, int inputSize); -/* -LZ4_compressHC : - return : the number of bytes in compressed buffer dest - or 0 if compression fails. - note : destination buffer must be already allocated. - To avoid any problem, size it to handle worst cases situations (input data not compressible) - Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h") -*/ -int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +/************************************** +* Block Compression +**************************************/ +int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); /* -LZ4_compress_limitedOutput() : - Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. - If it cannot achieve it, compression will stop, and result of the function will be zero. - This function never writes outside of provided output buffer. - - inputSize : Max supported value is 1 GB - maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated) - return : the number of output bytes written in buffer 'dest' - or 0 if compression fails. +LZ4_compress_HC : + Destination buffer 'dst' must be already allocated. + Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible) + Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h") + srcSize : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + compressionLevel : Recommended values are between 4 and 9, although any value between 0 and 16 will work. + 0 means "use default value" (see lz4hc.c). + Values >16 behave the same as 16. + return : the number of bytes written into buffer 'dst' + or 0 if compression fails. */ -int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); -/* - Same functions as above, but with programmable 'compressionLevel'. - Recommended values are between 4 and 9, although any value between 0 and 16 will work. - 'compressionLevel'==0 means use default 'compressionLevel' value. - Values above 16 behave the same as 16. - Equivalent variants exist for all other compression functions below. -*/ - /* Note : Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) */ -/************************************** -* Using an external allocation -**************************************/ int LZ4_sizeofStateHC(void); -int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); - -int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); - +int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); /* -These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. -To know how much memory must be allocated for the compression tables, use : -int LZ4_sizeofStateHC(); +LZ4_compress_HC_extStateHC() : + Use this function if you prefer to manually allocate memory for compression tables. + To know how much memory must be allocated for the compression tables, use : + int LZ4_sizeofStateHC(); -Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). + Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). -The allocated memory can be provided to the compression functions using 'void* state' parameter. -LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. -They just use the externally allocated memory for state instead of allocating their own (on stack, or on heap). + The allocated memory can then be provided to the compression functions using 'void* state' parameter. + LZ4_compress_HC_extStateHC() is equivalent to previously described function. + It just uses externally allocated memory for stateHC. */ - -/***************************** -* Includes -*****************************/ -#include <stddef.h> /* size_t */ - - /************************************** -* Experimental Streaming Functions +* Streaming Compression **************************************/ #define LZ4_STREAMHCSIZE 262192 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t; /* -LZ4_streamHC_t -This structure allows static allocation of LZ4 HC streaming state. -State must then be initialized using LZ4_resetStreamHC() before first use. + LZ4_streamHC_t + This structure allows static allocation of LZ4 HC streaming state. + State must then be initialized using LZ4_resetStreamHC() before first use. -Static allocation should only be used with statically linked library. -If you want to use LZ4 as a DLL, please use construction functions below, which are more future-proof. + Static allocation should only be used in combination with static linking. + If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof. */ LZ4_streamHC_t* LZ4_createStreamHC(void); -int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr); +int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); /* -These functions create and release memory for LZ4 HC streaming state. -Newly created states are already initialized. -Existing state space can be re-used anytime using LZ4_resetStreamHC(). -If you use LZ4 as a DLL, please use these functions instead of direct struct allocation, -to avoid size mismatch between different versions. + These functions create and release memory for LZ4 HC streaming state. + Newly created states are already initialized. + Existing state space can be re-used anytime using LZ4_resetStreamHC(). + If you use LZ4 as a DLL, use these functions instead of static structure allocation, + to avoid size mismatch between different versions. */ -void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); -int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize); +void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); +int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); -int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); -int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); -int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize); +int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); /* -These functions compress data in successive blocks of any size, using previous blocks as dictionary. -One key assumption is that each previous block will remain read-accessible while compressing next block. - -Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). -A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). - -Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block. -They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression. -Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. - -If, for any reason, previous data block can't be preserved in memory during next compression block, -you must save it to a safer memory space, -using LZ4_saveDictHC(). + These functions compress data in successive blocks of any size, using previous blocks as dictionary. + One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. + There is an exception for ring buffers, which can be smaller 64 KB. + Such case is automatically detected and correctly handled by LZ4_compress_HC_continue(). + + Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). + A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). + + Then, use LZ4_compress_HC_continue() to compress each successive block. + It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression. + Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. + As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation. + + If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, + you must save it to a safer memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. */ /************************************** - * Deprecated Streaming Functions - * ************************************/ -/* Note : these streaming functions follows the older model, and should no longer be used */ -void* LZ4_createHC (const char* inputBuffer); -char* LZ4_slideInputBufferHC (void* LZ4HC_Data); -int LZ4_freeHC (void* LZ4HC_Data); - -int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); -int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); - -int LZ4_sizeofStreamStateHC(void); -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); +* Deprecated Functions +**************************************/ +/* Deprecate Warnings */ +/* Should these warnings messages be a problem, + it is generally possible to disable them, + with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual for example. + You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ +#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ4_GCC_VERSION >= 301) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# define LZ4_DEPRECATED(message) +# endif +#endif // LZ4_DEPRECATE_WARNING_DEFBLOCK + +/* compression functions */ +/* these functions are planned to trigger warning messages by r131 approximately */ +int LZ4_compressHC (const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); +int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); +int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Streaming functions following the older model; should no longer be used */ +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); +LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); +LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); #if defined (__cplusplus) diff --git a/lib/xxhash.c b/lib/xxhash.c index aca1e0a..e6fb8f1 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -29,13 +29,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - xxHash source repository : https://github.com/Cyan4973/xxHash -- public discussion board : https://groups.google.com/forum/#!forum/lz4c */ /************************************** * Tuning parameters -***************************************/ +**************************************/ /* Unaligned memory access is automatically enabled for "common" CPU, such as x86. * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. @@ -93,10 +92,7 @@ static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } /* for memcpy() */ #include <string.h> -static void* XXH_memcpy(void* dest, const void* src, size_t size) -{ - return memcpy(dest,src,size); -} +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } /************************************** @@ -104,51 +100,36 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) ***************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include <stdint.h> -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; #else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; #endif -typedef struct _U32_S +static U32 XXH_read32(const void* memPtr) { - U32 v; -} _PACKED U32_S; -typedef struct _U64_S -{ - U64 v; -} _PACKED U64_S; + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif +static U64 XXH_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; +} -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) -/***************************************** +/****************************************** * Compiler-specific Functions and Macros ******************************************/ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) @@ -190,39 +171,17 @@ static U64 XXH_swap64 (U64 x) #endif -/************************************** -* Constants -***************************************/ -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - -#define PRIME64_1 11400714785074694791ULL -#define PRIME64_2 14029467366897019727ULL -#define PRIME64_3 1609587929392839161ULL -#define PRIME64_4 9650029242287828579ULL -#define PRIME64_5 2870177450012600261ULL - - /*************************************** * Architecture Macros -****************************************/ +***************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; #ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) #endif -/************************************** -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ - - -/**************************** +/***************************** * Memory reads *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; @@ -230,9 +189,9 @@ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); else - return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) @@ -243,9 +202,9 @@ FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); else - return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) @@ -254,7 +213,29 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) } -/**************************** +/*************************************** +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ + + +/*************************************** +* Constants +***************************************/ +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + + +/***************************** * Simple Hash Functions *****************************/ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) @@ -334,7 +315,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH } -unsigned int XXH32 (const void* input, size_t len, unsigned seed) +unsigned XXH32 (const void* input, size_t len, unsigned seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ @@ -346,7 +327,7 @@ unsigned int XXH32 (const void* input, size_t len, unsigned seed) XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == 0) /* Input is aligned, let's leverage the speed advantage */ + if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -503,7 +484,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed } /**************************************************** - * Advanced Hash Functions +* Advanced Hash Functions ****************************************************/ /*** Allocation ***/ @@ -687,9 +668,9 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) { - XXH_istate32_t* state = (XXH_istate32_t*) state_in; + const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; const BYTE * p = (const BYTE*)state->mem32; - BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; + const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->total_len >= 16) @@ -841,9 +822,9 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) { - XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; const BYTE * p = (const BYTE*)state->mem64; - BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; + const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) diff --git a/lib/xxhash.h b/lib/xxhash.h index 34eea73..0df5ac1 100644 --- a/lib/xxhash.h +++ b/lib/xxhash.h @@ -29,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ + - xxHash source repository : https://github.com/Cyan4973/xxHash */ /* Notice extracted from xxHash homepage : @@ -57,8 +57,8 @@ Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. -A new 64-bits version, named XXH64, is available since r35. -It offers better speed for 64-bits applications. +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. Name Speed on 64 bits Speed on 32 bits XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s diff --git a/lz4_block_format.txt b/lz4_Block_format.md index e248fd9..a120a0b 100644 --- a/lz4_block_format.txt +++ b/lz4_Block_format.md @@ -1,10 +1,9 @@ LZ4 Block Format Description ============================ -Last revised: 2015-03-26; +Last revised: 2015-03-26. Author : Yann Collet - This small specification intents to provide enough information to anyone willing to produce LZ4-compatible compressed data blocks using any programming language. @@ -26,7 +25,8 @@ on implementation details of the compressor, and vice versa. Compressed block format ----------------------- An LZ4 compressed block is composed of sequences. -Schematically, a sequence is a suite of literals, followed by a match copy. +A sequence is a suite of literals (not-compressed bytes), +followed by a match copy. Each sequence starts with a token. The token is a one byte value, separated into two 4-bits fields. @@ -35,27 +35,30 @@ Therefore each field ranges from 0 to 15. The first field uses the 4 high-bits of the token. It provides the length of literals to follow. -(Note : a literal is a not-compressed byte). + If the field value is 0, then there is no literal. If it is 15, then we need to add some more bytes to indicate the full length. -Each additionnal byte then represent a value from 0 to 255, +Each additional byte then represent a value from 0 to 255, which is added to the previous value to produce a total length. When the byte value is 255, another byte is output. There can be any number of bytes following the token. There is no "size limit". -(Sidenote this is why a not-compressible input block is expanded by 0.4%). +(Side note : this is why a not-compressible input block is expanded by 0.4%). Example 1 : A length of 48 will be represented as : -- 15 : value for the 4-bits High field -- 33 : (=48-15) remaining length to reach 48 + + - 15 : value for the 4-bits High field + - 33 : (=48-15) remaining length to reach 48 Example 2 : A length of 280 will be represented as : -- 15 : value for the 4-bits High field -- 255 : following byte is maxed, since 280-15 >= 255 -- 10 : (=280 - 15 - 255) ) remaining length to reach 280 + + - 15 : value for the 4-bits High field + - 255 : following byte is maxed, since 280-15 >= 255 + - 10 : (=280 - 15 - 255) ) remaining length to reach 280 Example 3 : A length of 15 will be represented as : -- 15 : value for the 4-bits High field -- 0 : (=15-15) yes, the zero must be output + + - 15 : value for the 4-bits High field + - 0 : (=15-15) yes, the zero must be output Following the token and optional length bytes, are the literals themselves. They are exactly as numerous as previously decoded (length of literals). @@ -65,7 +68,8 @@ It's possible that there are zero literal. Following the literals is the match copy operation. It starts by the offset. -This is a 2 bytes value, in little endian format. +This is a 2 bytes value, in little endian format +(the 1st byte is the "low" byte, the 2nd one is the "high" byte). The offset represents the position of the match to be copied from. 1 means "current position - 1 byte". @@ -83,7 +87,7 @@ we output additional bytes, one at a time, with values ranging from 0 to 255. They are added to total to provide the final match length. A 255 value means there is another byte to read and add. There is no limit to the number of optional bytes that can be output this way. -(This points towards a maximum achievable compression ratio of ~250). +(This points towards a maximum achievable compression ratio of about 250). With the offset and the matchlength, the decoder can now proceed to copy the data from the already decoded buffer. @@ -95,9 +99,11 @@ Parsing restrictions ----------------------- There are specific parsing rules to respect in order to remain compatible with assumptions made by the decoder : -1) The last 5 bytes are always literals -2) The last match must start at least 12 bytes before end of block -Consequently, a block with less than 13 bytes cannot be compressed. + +1. The last 5 bytes are always literals +2. The last match must start at least 12 bytes before end of block. + Consequently, a block with less than 13 bytes cannot be compressed. + These rules are in place to ensure that the decoder will never read beyond the input buffer, nor write beyond the output buffer. @@ -118,4 +124,3 @@ or full optimal parsing. All these trade-off offer distinctive speed/memory/compression advantages. Whatever the method used by the compressor, its result will be decodable by any LZ4 decoder if it follows the format specification described above. - diff --git a/lz4_Frame_format.md b/lz4_Frame_format.md new file mode 100644 index 0000000..cb41d47 --- /dev/null +++ b/lz4_Frame_format.md @@ -0,0 +1,385 @@ +LZ4 Frame Format Description +============================ + +###Notices + +Copyright (c) 2013-2015 Yann Collet + +Permission is granted to copy and distribute this document +for any purpose and without charge, +including translations into other languages +and incorporation into compilations, +provided that the copyright notice and this notice are preserved, +and that any substantive changes or deletions from the original +are clearly marked. +Distribution of this document is unlimited. + +###Version + +1.5.1 (31/03/2015) + + +Introduction +------------ + +The purpose of this document is to define a lossless compressed data format, +that is independent of CPU type, operating system, +file system and character set, suitable for +File compression, Pipe and streaming compression +using the [LZ4 algorithm](http://www.lz4.info). + +The data can be produced or consumed, +even for an arbitrarily long sequentially presented input data stream, +using only an a priori bounded amount of intermediate storage, +and hence can be used in data communications. +The format uses the LZ4 compression method, +and optional [xxHash-32 checksum method](https://github.com/Cyan4973/xxHash), +for detection of data corruption. + +The data format defined by this specification +does not attempt to allow random access to compressed data. + +This specification is intended for use by implementers of software +to compress data into LZ4 format and/or decompress data from LZ4 format. +The text of the specification assumes a basic background in programming +at the level of bits and other primitive data representations. + +Unless otherwise indicated below, +a compliant compressor must produce data sets +that conform to the specifications presented here. +It doesn’t need to support all options though. + +A compliant decompressor must be able to decompress +at least one working set of parameters +that conforms to the specifications presented here. +It may also ignore checksums. +Whenever it does not support a specific parameter within the compressed stream, +it must produce a non-ambiguous error code +and associated error message explaining which parameter is unsupported. + + +General Structure of LZ4 Frame format +------------------------------------- + +| MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum | +|:-------:|:-------------:| ----- | ----- | ------- | ----------- | +| 4 bytes | 3-11 bytes | | | 4 bytes | 4 bytes | + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184D2204 + +__Frame Descriptor__ + +3 to 11 Bytes, to be detailed in the next part. +Most important part of the spec. + +__Data Blocks__ + +To be detailed later on. +That’s where compressed data is stored. + +__EndMark__ + +The flow of blocks ends when the last data block has a size of “0”. +The size is expressed as a 32-bits value. + +__Content Checksum__ + +Content Checksum verify that the full content has been decoded correctly. +The content checksum is the result +of [xxh32() hash function](https://github.com/Cyan4973/xxHash) +digesting the original (decoded) data as input, and a seed of zero. +Content checksum is only present when its associated flag +is set in the frame descriptor. +Content Checksum validates the result, +that all blocks were fully transmitted in the correct order and without error, +and also that the encoding/decoding process itself generated no distortion. +Its usage is recommended. + +__Frame Concatenation__ + +In some circumstances, it may be preferable to append multiple frames, +for example in order to add new data to an existing compressed file +without re-framing it. + +In such case, each frame has its own set of descriptor flags. +Each frame is considered independent. +The only relation between frames is their sequential order. + +The ability to decode multiple concatenated frames +within a single stream or file +is left outside of this specification. +As an example, the reference lz4 command line utility behavior is +to decode all concatenated frames in their sequential order. + + +Frame Descriptor +---------------- + +| FLG | BD | (Content Size) | HC | +| ------- | ------- |:--------------:| ------- | +| 1 byte | 1 byte | 0 - 8 bytes | 1 byte | + +The descriptor uses a minimum of 3 bytes, +and up to 11 bytes depending on optional parameters. + +__FLG byte__ + +| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 | +| ------- | ------- | ------- | --------- | ------- | --------- | -------- | +|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*| + + +__BD byte__ + +| BitNb | 7 | 6-5-4 | 3-2-1-0 | +| ------- | -------- | ------------ | -------- | +|FieldName|*Reserved*| Block MaxSize|*Reserved*| + +In the tables, bit 7 is highest bit, while bit 0 is lowest. + +__Version Number__ + +2-bits field, must be set to “01”. +Any other value cannot be decoded by this version of the specification. +Other version numbers will use different flag layouts. + +__Block Independence flag__ + +If this flag is set to “1”, blocks are independent. +If this flag is set to “0”, each block depends on previous ones +(up to LZ4 window size, which is 64 KB). +In such case, it’s necessary to decode all blocks in sequence. + +Block dependency improves compression ratio, especially for small blocks. +On the other hand, it makes direct jumps or multi-threaded decoding impossible. + +__Block checksum flag__ + +If this flag is set, each data block will be followed by a 4-bytes checksum, +calculated by using the xxHash-32 algorithm on the raw (compressed) data block. +The intention is to detect data corruption (storage or transmission errors) +immediately, before decoding. +Block checksum usage is optional. + +__Content Size flag__ + +If this flag is set, the uncompressed size of data included within the frame +will be present as an 8 bytes unsigned little endian value, after the flags. +Content Size usage is optional. + +__Content checksum flag__ + +If this flag is set, a content checksum will be appended after the EndMark. + +Recommended value : “1” (content checksum is present) + +__Block Maximum Size__ + +This information is intended to help the decoder allocate memory. +Size here refers to the original (uncompressed) data size. +Block Maximum Size is one value among the following table : + +| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | +| --- | --- | --- | --- | ----- | ------ | ---- | ---- | +| N/A | N/A | N/A | N/A | 64 KB | 256 KB | 1 MB | 4 MB | + +The decoder may refuse to allocate block sizes above a (system-specific) size. +Unused values may be used in a future revision of the spec. +A decoder conformant to the current version of the spec +is only able to decode blocksizes defined in this spec. + +__Reserved bits__ + +Value of reserved bits **must** be 0 (zero). +Reserved bit might be used in a future version of the specification, +typically enabling new optional features. +If this happens, a decoder respecting the current version of the specification +shall not be able to decode such a frame. + +__Content Size__ + +This is the original (uncompressed) size. +This information is optional, and only present if the associated flag is set. +Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes. +Format is Little endian. +This value is informational. +It can be safely skipped by a conformant decoder. + +__Header Checksum__ + +One-byte checksum of combined descriptor fields, including optional ones. +The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF } ` +using zero as a seed, +and the full Frame Descriptor as an input +(including optional fields when they are present). +A wrong checksum indicates an error in the descriptor. +Header checksum is informational and can be skipped. + + +Data Blocks +----------- + +| Block Size | data | (Block Checksum) | +|:----------:| ------ |:----------------:| +| 4 bytes | | 0 - 4 bytes | + + +__Block Size__ + +This field uses 4-bytes, format is little-endian. + +The highest bit is “1” if data in the block is uncompressed. + +The highest bit is “0” if data in the block is compressed by LZ4. + +All other bits give the size, in bytes, of the following data block +(the size does not include the block checksum if present). + +Block Size shall never be larger than Block Maximum Size. +Such a thing could happen for incompressible source data. +In such case, such a data block shall be passed in uncompressed format. + +__Data__ + +Where the actual data to decode stands. +It might be compressed or not, depending on previous field indications. +Uncompressed size of Data can be any size, up to “block maximum size”. +Note that data block is not necessarily full : +an arbitrary “flush” may happen anytime. Any block can be “partially filled”. + +__Block checksum__ + +Only present if the associated flag is set. +This is a 4-bytes checksum value, in little endian format, +calculated by using the xxHash-32 algorithm on the raw (undecoded) data block, +and a seed of zero. +The intention is to detect data corruption (storage or transmission errors) +before decoding. + +Block checksum is cumulative with Content checksum. + + +Skippable Frames +---------------- + +| Magic Number | Frame Size | User Data | +|:------------:|:----------:| --------- | +| 4 bytes | 4 bytes | | + +Skippable frames allow the integration of user-defined data +into a flow of concatenated frames. +Its design is pretty straightforward, +with the sole objective to allow the decoder to quickly skip +over user-defined data and continue decoding. + +For the purpose of facilitating identification, +it is discouraged to start a flow of concatenated frames with a skippable frame. +If there is a need to start such a flow with some user data +encapsulated into a skippable frame, +it’s recommended to start with a zero-byte LZ4 frame +followed by a skippable frame. +This will make it easier for file type identifiers. + + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. +All 16 values are valid to identify a skippable frame. + +__Frame Size__ + +This is the size, in bytes, of the following User Data +(without including the magic number nor the size field itself). +4 Bytes, Little endian format, unsigned 32-bits. +This means User Data can’t be bigger than (2^32-1) Bytes. + +__User Data__ + +User Data can be anything. Data will just be skipped by the decoder. + + +Legacy frame +------------ + +The Legacy frame format was defined into the initial versions of “LZ4Demo”. +Newer compressors should not use this format anymore, as it is too restrictive. + +Main characteristics of the legacy format : + +- Fixed block size : 8 MB. +- All blocks must be completely filled, except the last one. +- All blocks are always compressed, even when compression is detrimental. +- The last block is detected either because + it is followed by the “EOF” (End of File) mark, + or because it is followed by a known Frame Magic Number. +- No checksum +- Convention is Little endian + +| MagicNb | B.CSize | CData | B.CSize | CData | (...) | EndMark | +| ------- | ------- | ----- | ------- | ----- | ------- | ------- | +| 4 bytes | 4 bytes | CSize | 4 bytes | CSize | x times | EOF | + + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184C2102 + +__Block Compressed Size__ + +This is the size, in bytes, of the following compressed data block. +4 Bytes, Little endian format. + +__Data__ + +Where the actual compressed data stands. +Data is always compressed, even when compression is detrimental. + +__EndMark__ + +End of legacy frame is implicit only. +It must be followed by a standard EOF (End Of File) signal, +wether it is a file or a stream. + +Alternatively, if the frame is followed by a valid Frame Magic Number, +it is considered completed. +It makes legacy frames compatible with frame concatenation. + +Any other value will be interpreted as a block size, +and trigger an error if it does not fit within acceptable range. + + +Version changes +--------------- + +1.5.1 : changed format to MarkDown compatible + +1.5 : removed Dictionary ID from specification + +1.4.1 : changed wording from “stream” to “frame” + +1.4 : added skippable streams, re-added stream checksum + +1.3 : modified header checksum + +1.2 : reduced choice of “block size”, to postpone decision on “dynamic size of BlockSize Field”. + +1.1 : optional fields are now part of the descriptor + +1.0 : changed “block size” specification, adding a compressed/uncompressed flag + +0.9 : reduced scale of “block maximum size” table + +0.8 : removed : high compression flag + +0.7 : removed : stream checksum + +0.6 : settled : stream size uses 8 bytes, endian convention is little endian + +0.5: added copyright notice + +0.4 : changed format to Google Doc compatible OpenDocument
\ No newline at end of file diff --git a/programs/Makefile b/programs/Makefile index a324148..39335db 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,24 +27,24 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # fuzzer : Test tool, to check lz4 integrity on target platform # fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# frametest : Test tool, to check lz4frame integrity on target platform +# frametest32: Same as frametest, but forced to compile in 32-bits mode # fullbench : Precisely measure speed for each LZ4 function variant # fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# datagen : generates synthetic data samples for tests & benchmarks # ########################################################################## -RELEASE?= r128 +RELEASE?= r129 DESTDIR?= PREFIX ?= /usr/local CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\" -FLAGS = -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\" +FLAGS := -I../lib $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -BINDIR=$(PREFIX)/bin -MANDIR=$(PREFIX)/share/man/man1 -LZ4DIR=../lib - -TEST_FILES = COPYING -TEST_TARGETS=test-native +BINDIR := $(PREFIX)/bin +MANDIR := $(PREFIX)/share/man/man1 +LZ4DIR := ../lib # Define *.exe as extension for Windows systems @@ -58,7 +58,10 @@ endif # Select test target for Travis CI's Build Matrix -TRAVIS_TARGET=$(LZ4_TRAVIS_CI_ENV) +TRAVIS_TARGET:= $(LZ4_TRAVIS_CI_ENV) +TEST_FILES := COPYING +TEST_TARGETS := test-native +FUZZER_TIME := -T9mn default: lz4 @@ -100,7 +103,7 @@ datagen : datagen.c datagencli.c $(CC) $(FLAGS) $^ -o $@$(EXT) clean: - @rm -f core *.o *.test \ + @rm -f core *.o *.test tmp* \ lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) \ @@ -148,15 +151,18 @@ test-travis: $(TRAVIS_TARGET) test-lz4-sparse: lz4 datagen @echo ---- test sparse file support ---- - ./datagen -g50M -P100 | ./lz4 -B4D | ./lz4 -dv --sparse > tmpB4 - ./datagen -g50M -P100 | ./lz4 -B5D | ./lz4 -dv --sparse > tmpB5 - ./datagen -g50M -P100 | ./lz4 -B6D | ./lz4 -dv --sparse > tmpB6 - ./datagen -g50M -P100 | ./lz4 -B7D | ./lz4 -dv --sparse > tmpB7 + ./datagen -g5M -P100 > tmpSrc + ./lz4 -B4D tmpSrc | ./lz4 -dv --sparse > tmpB4 + diff -s tmpSrc tmpB4 + ./lz4 -B5D tmpSrc | ./lz4 -dv --sparse > tmpB5 + diff -s tmpSrc tmpB5 + ./lz4 -B6D tmpSrc | ./lz4 -dv --sparse > tmpB6 + diff -s tmpSrc tmpB6 + ./lz4 -B7D tmpSrc | ./lz4 -dv --sparse > tmpB7 + diff -s tmpSrc tmpB7 + ./lz4 tmpSrc | ./lz4 -dv --no-sparse > tmpNoSparse + diff -s tmpSrc tmpNoSparse ls -ls tmp* - ./datagen -g50M -P100 | diff -s - tmpB4 - ./datagen -g50M -P100 | diff -s - tmpB5 - ./datagen -g50M -P100 | diff -s - tmpB6 - ./datagen -g50M -P100 | diff -s - tmpB7 ./datagen -s1 -g1200007 -P100 | ./lz4 | ./lz4 -dv --sparse > tmpOdd # Odd size file (to not finish on an exact nb of blocks) ./datagen -s1 -g1200007 -P100 | diff -s - tmpOdd ls -ls tmpOdd @@ -168,6 +174,11 @@ test-lz4-contentSize: lz4 datagen ./lz4 -v tmp | ./lz4 -t ./lz4 -v --content-size tmp | ./lz4 -d > tmp2 diff -s tmp tmp2 + # test large size [2-4] GB + @./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp + @ls -ls tmp + ./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2 + @ls -ls tmp2 @rm tmp* test-lz4-frame-concatenation: lz4 datagen @@ -183,24 +194,30 @@ test-lz4-frame-concatenation: lz4 datagen @rm *.test @echo frame concatenation test completed -test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation +test-lz4-multiple: lz4 datagen + @echo ---- test multiple files ---- + @./datagen -s1 > tmp1 2> $(VOID) + @./datagen -s2 -g100K > tmp2 2> $(VOID) + @./datagen -s3 -g1M > tmp3 2> $(VOID) + ./lz4 -f -m tmp* + ls -ls tmp* + rm tmp1 tmp2 tmp3 + ./lz4 -df -m *.lz4 + ls -ls tmp* + ./lz4 -f -m tmp1 notHere tmp2; echo $$? + @rm tmp* + +test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation @echo ---- test lz4 basic compression/decompression ---- ./datagen -g0 | ./lz4 -v | ./lz4 -t ./datagen -g16KB | ./lz4 -9 | ./lz4 -t ./datagen | ./lz4 | ./lz4 -t ./datagen -g6M -P99 | ./lz4 -9BD | ./lz4 -t - ./datagen -g17M | ./lz4 -9v | ./lz4 -tq + ./datagen -g17M | ./lz4 -9v | ./lz4 -qt ./datagen -g33M | ./lz4 --no-frame-crc | ./lz4 -t ./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t - ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -t - ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -t - @echo ---- test multiple input files ---- - @./datagen -s1 > file1 - @./datagen -s2 > file2 - @./datagen -s3 > file3 - ./lz4 -f -m file1 file2 file3 - ls -l file* - @rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4 + ./datagen -g6GB | ./lz4 -vqB5D | ./lz4 -qt + ./datagen -g6GB | ./lz4 -vq9BD | ./lz4 -qt @echo ---- test pass-through ---- ./datagen | ./lz4 -tf @@ -212,41 +229,46 @@ test-lz4c32: lz4 lz4c32 datagen ./datagen -g16KB | ./lz4c32 -9 | ./lz4 -t ./datagen | ./lz4c32 | ./lz4c32 -t ./datagen | ./lz4c32 | ./lz4 -t - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -t - ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -t - ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -t - ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -t + ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4c32 -qt + ./datagen -g256MB | ./lz4c32 -vqB4D | ./lz4 -qt + ./datagen -g6GB | ./lz4c32 -vqB5D | ./lz4c32 -qt + ./datagen -g6GB | ./lz4c32 -vq9BD | ./lz4 -qt test-fullbench: fullbench - ./fullbench --no-prompt $(TEST_FILES) + ./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES) test-fullbench32: fullbench32 - ./fullbench32 --no-prompt $(TEST_FILES) + ./fullbench32 --no-prompt $(NB_LOOPS) $(TEST_FILES) test-fuzzer: fuzzer - ./fuzzer + ./fuzzer $(FUZZER_TIME) test-fuzzer32: fuzzer32 - ./fuzzer32 + ./fuzzer32 $(FUZZER_TIME) test-frametest: frametest - ./frametest + ./frametest $(FUZZER_TIME) test-frametest32: frametest32 - ./frametest32 + ./frametest32 $(FUZZER_TIME) -test-mem: lz4 datagen fuzzer frametest - valgrind --leak-check=yes ./datagen -g50M > $(VOID) +test-mem: lz4 datagen fuzzer frametest fullbench + valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID) ./datagen -g16KB > tmp - valgrind --leak-check=yes ./lz4 -9 -BD -f tmp $(VOID) + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -BD -f tmp $(VOID) + ./datagen -g16KB -s2 > tmp2 + ./datagen -g16KB -s3 > tmp3 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 --force --multiple tmp tmp2 tmp3 ./datagen -g16MB > tmp - valgrind --leak-check=yes ./lz4 -9 -B5D -f tmp tmp2 - valgrind --leak-check=yes ./lz4 -t tmp2 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -9 -B5D -f tmp tmp2 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -t tmp2 + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -bi1 tmp + valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 tmp tmp2 ./datagen -g256MB > tmp - valgrind --leak-check=yes ./lz4 -B4D -f -vq tmp $(VOID) + valgrind --leak-check=yes --error-exitcode=1 ./lz4 -B4D -f -vq tmp $(VOID) rm tmp* - valgrind --leak-check=yes ./fuzzer -i64 -t1 - valgrind --leak-check=yes ./frametest -i256 + valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1 + valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256 test-mem32: lz4c32 datagen # unfortunately, valgrind doesn't seem to work with non-native binary. If someone knows how to do a valgrind-test on a 32-bits exe with a 64-bits system... diff --git a/programs/bench.c b/programs/bench.c index e1b5357..9f949c4 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -58,9 +58,9 @@ #include "lz4.h" #define COMPRESSOR0 LZ4_compress_local -static int LZ4_compress_local(const char* src, char* dst, int size, int clevel) { (void)clevel; return LZ4_compress(src, dst, size); } +static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSize, int clevel) { (void)clevel; return LZ4_compress_default(src, dst, srcSize, dstSize); } #include "lz4hc.h" -#define COMPRESSOR1 LZ4_compressHC2 +#define COMPRESSOR1 LZ4_compress_HC #define DEFAULTCOMPRESSOR COMPRESSOR0 #include "xxhash.h" @@ -121,8 +121,8 @@ struct chunkParameters struct compressionParameters { - int (*compressionFunction)(const char*, char*, int, int); - int (*decompressionFunction)(const char*, char*, int); + int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); + int (*decompressionFunction)(const char* src, char* dst, int dstSize); }; @@ -281,17 +281,13 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) /* Check file existence */ inFileName = fileNamesTable[fileIdx++]; inFile = fopen( inFileName, "rb" ); - if (inFile==NULL) - { - DISPLAY( "Pb opening %s\n", inFileName); - return 11; - } + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } /* Memory allocation & restrictions */ inFileSize = BMK_GetFileSize(inFileName); - if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; } + if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } benchedSize = (size_t) BMK_findMaxMem(inFileSize * 2) / 2; - if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; } + if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) { @@ -306,7 +302,6 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) compressedBuffSize = nbChunks * maxCompressedChunkSize; compressedBuffer = (char*)malloc((size_t)compressedBuffSize); - if (!orig_buff || !compressedBuffer) { DISPLAY("\nError: not enough memory!\n"); @@ -376,11 +371,12 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { for (chunkNb=0; chunkNb<nbChunks; chunkNb++) - chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, cLevel); + chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel); nbLoops++; } milliTime = BMK_GetMilliSpan(milliTime); + nbLoops += !nbLoops; /* avoid division by zero */ if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime/nbLoops; cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; ratio = (double)cSize/(double)benchedSize*100.; @@ -402,8 +398,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) } milliTime = BMK_GetMilliSpan(milliTime); + nbLoops += !nbLoops; /* avoid division by zero */ if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime/nbLoops; - DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); /* CRC Checking */ crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0); @@ -413,9 +410,9 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) if (crcOrig==crcCheck) { if (ratio<100.) - DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); else - DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); } totals += benchedSize; totalz += cSize; @@ -431,7 +428,7 @@ int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel) if (nbFiles > 1) DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.); - if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); } + if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); } return 0; } diff --git a/programs/frametest.c b/programs/frametest.c index 4beee23..46ec030 100644 --- a/programs/frametest.c +++ b/programs/frametest.c @@ -238,10 +238,11 @@ int basicTests(U32 seed, double compressibility) U32 randState = seed; size_t cSize, testSize; LZ4F_preferences_t prefs; - LZ4F_decompressionContext_t dCtx; + LZ4F_decompressionContext_t dCtx = NULL; + LZ4F_compressionContext_t cctx = NULL; U64 crcOrig; - // Create compressible test buffer + /* Create compressible test buffer */ memset(&prefs, 0, sizeof(prefs)); CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); compressedBuffer = malloc(LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL)); @@ -249,7 +250,7 @@ int basicTests(U32 seed, double compressibility) FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); - // Trivial tests : one-step frame + /* Trivial tests : one-step frame */ testSize = COMPRESSIBLE_NOISE_LENGTH; DISPLAYLEVEL(3, "Using NULL preferences : \n"); cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL); @@ -276,12 +277,58 @@ int basicTests(U32 seed, double compressibility) if (crcDest != crcOrig) goto _output_error; DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + DISPLAYLEVEL(4, "Reusing decompression context \n"); + { + size_t iSize = compressedBufferSize - 4; + const BYTE* cBuff = (const BYTE*) compressedBuffer; + DISPLAYLEVEL(3, "Missing last 4 bytes : "); + errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + if (!errorCode) goto _output_error; + DISPLAYLEVEL(3, "indeed, request %u bytes \n", (unsigned)errorCode); + cBuff += iSize; + iSize = errorCode; + errorCode = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, cBuff, &iSize, NULL); + if (errorCode != 0) goto _output_error; + crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); + if (crcDest != crcOrig) goto _output_error; + } + + { + size_t oSize = 0; + size_t iSize = 0; + LZ4F_frameInfo_t fi; + + DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : "); + errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL); + if (LZ4F_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode); + + DISPLAYLEVEL(3, "get FrameInfo on null input : "); + errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); + if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; + DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); + + DISPLAYLEVEL(3, "get FrameInfo on not enough input : "); + iSize = 6; + errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); + if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error; + DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode)); + ip += iSize; + + DISPLAYLEVEL(3, "get FrameInfo on enough input : "); + iSize = 15 - iSize; + errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize); + if (LZ4F_isError(errorCode)) goto _output_error; + DISPLAYLEVEL(3, " correctly decoded \n"); + ip += iSize; + } + DISPLAYLEVEL(3, "Byte after byte : \n"); while (ip < iend) { size_t oSize = oend-op; size_t iSize = 1; - //DISPLAY("%7i \n", (int)(ip-(BYTE*)compressedBuffer)); errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL); if (LZ4F_isError(errorCode)) goto _output_error; op += oSize; @@ -289,28 +336,28 @@ int basicTests(U32 seed, double compressibility) } crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1); if (crcDest != crcOrig) goto _output_error; - DISPLAYLEVEL(3, "Regenerated %i bytes \n", (int)decodedBufferSize); + DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-(BYTE*)decodedBuffer), COMPRESSIBLE_NOISE_LENGTH); errorCode = LZ4F_freeDecompressionContext(dCtx); if (LZ4F_isError(errorCode)) goto _output_error; } DISPLAYLEVEL(3, "Using 64 KB block : \n"); - prefs.frameInfo.blockSizeID = max64KB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max64KB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 256 KB block : \n"); - prefs.frameInfo.blockSizeID = max256KB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max256KB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); @@ -350,33 +397,33 @@ int basicTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 1 MB block : \n"); - prefs.frameInfo.blockSizeID = max1MB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max1MB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "Using 4 MB block : \n"); - prefs.frameInfo.blockSizeID = max4MB; - prefs.frameInfo.contentChecksumFlag = contentChecksumEnabled; + prefs.frameInfo.blockSizeID = LZ4F_max4MB; + prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); DISPLAYLEVEL(3, "without checksum : \n"); - prefs.frameInfo.contentChecksumFlag = noContentChecksum; + prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum; cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs); if (LZ4F_isError(cSize)) goto _output_error; DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize); @@ -385,7 +432,6 @@ int basicTests(U32 seed, double compressibility) size_t errorCode; BYTE* const ostart = (BYTE*)compressedBuffer; BYTE* op = ostart; - LZ4F_compressionContext_t cctx; errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) goto _output_error; @@ -430,6 +476,7 @@ int basicTests(U32 seed, double compressibility) errorCode = LZ4F_freeCompressionContext(cctx); if (LZ4F_isError(errorCode)) goto _output_error; + cctx = NULL; } DISPLAYLEVEL(3, "Skippable frame test : \n"); @@ -500,10 +547,6 @@ int basicTests(U32 seed, double compressibility) ip += iSize; } DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8)); - - /* release memory */ - errorCode = LZ4F_freeDecompressionContext(dCtx); - if (LZ4F_isError(errorCode)) goto _output_error; } DISPLAY("Basic tests completed \n"); @@ -511,6 +554,8 @@ _end: free(CNBuffer); free(compressedBuffer); free(decodedBuffer); + LZ4F_freeDecompressionContext(dCtx); dCtx = NULL; + LZ4F_freeCompressionContext(cctx); cctx = NULL; return testResult; _output_error: @@ -523,8 +568,8 @@ _output_error: static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous) { int p=0; - BYTE* b1=(BYTE*)buff1; - BYTE* b2=(BYTE*)buff2; + const BYTE* b1=(const BYTE*)buff1; + const BYTE* b2=(const BYTE*)buff2; if (nonContiguous) { DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size); @@ -537,7 +582,7 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, un static const U32 srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */ -int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility) +int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration) { unsigned testResult = 0; unsigned testNb = 0; @@ -548,10 +593,15 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi LZ4F_decompressionContext_t dCtx = NULL; LZ4F_compressionContext_t cCtx = NULL; size_t result; + const U32 startTime = FUZ_GetMilliStart(); XXH64_state_t xxh64; # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + + /* Init */ + duration *= 1000; + /* Create buffers */ result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result); @@ -566,10 +616,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand); /* jump to requested testNb */ - for (testNb =0; testNb < startTest; testNb++) (void)FUZ_rand(&coreRand); // sync randomizer + for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); // sync randomizer /* main fuzzer test loop */ - for ( ; testNb < nbTests; testNb++) + for ( ; (testNb < nbTests) || (duration > FUZ_GetMilliSpan(startTime)) ; testNb++) { U32 randState = coreRand ^ prime1; unsigned BSId = 4 + (FUZ_rand(&randState) & 3); @@ -591,9 +641,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi memset(&prefs, 0, sizeof(prefs)); memset(&cOptions, 0, sizeof(cOptions)); memset(&dOptions, 0, sizeof(dOptions)); - prefs.frameInfo.blockMode = (blockMode_t)BMId; - prefs.frameInfo.blockSizeID = (blockSizeID_t)BSId; - prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)CCflag; + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)BMId; + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)BSId; + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)CCflag; prefs.frameInfo.contentSize = frameContentSize; prefs.autoFlush = autoflush; prefs.compressionLevel = FUZ_rand(&randState) % 5; @@ -662,6 +712,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi unsigned nonContiguousDst = (FUZ_rand(&randState) & 3) == 1; nonContiguousDst += FUZ_rand(&randState) & nonContiguousDst; /* 0=>0; 1=>1,2 */ XXH64_reset(&xxh64, 1); + if (maxBits < 3) maxBits = 3; while (ip < iend) { unsigned nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1; @@ -673,7 +724,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi dOptions.stableDst = FUZ_rand(&randState) & 1; if (nonContiguousDst==2) dOptions.stableDst = 0; result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions); - if (result == (size_t)-ERROR_checksum_invalid) locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); + if (result == (size_t)-LZ4F_ERROR_contentChecksum_invalid) + locateBuffDiff((BYTE*)srcBuffer+srcStart, decodedBuffer, srcSize, nonContiguousDst); CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName((LZ4F_errorCode_t)result)); XXH64_update(&xxh64, op, (U32)oSize); totalOut += oSize; @@ -704,7 +756,7 @@ _end: if (pause) { DISPLAY("press enter to finish \n"); - getchar(); + (void)getchar(); } return testResult; @@ -721,6 +773,7 @@ int FUZ_usage(void) DISPLAY( "\n"); DISPLAY( "Arguments :\n"); DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); + DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -p# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); @@ -739,6 +792,7 @@ int main(int argc, char** argv) int testNb = 0; int proba = FUZ_COMPRESSIBILITY_DEFAULT; int result=0; + U32 duration=0; /* Check command line */ programName = argv[0]; @@ -746,9 +800,9 @@ int main(int argc, char** argv) { char* argument = argv[argNb]; - if(!argument) continue; // Protection if argument empty + if(!argument) continue; /* Protection if argument empty */ - // Decode command (note : aggregated commands are allowed) + /* Decode command (note : aggregated commands are allowed) */ if (argument[0]=='-') { if (!strcmp(argument, "--no-prompt")) @@ -781,7 +835,7 @@ int main(int argc, char** argv) case 'i': argument++; - nbTests=0; + nbTests=0; duration=0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; @@ -789,6 +843,32 @@ int main(int argc, char** argv) argument++; } break; + + case 'T': + argument++; + nbTests = 0; duration = 0; + for (;;) + { + switch(*argument) + { + case 'm': duration *= 60; argument++; continue; + case 's': + case 'n': argument++; continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': duration *= 10; duration += *argument++ - '0'; continue; + } + break; + } + break; + case 's': argument++; seed=0; @@ -841,5 +921,5 @@ int main(int argc, char** argv) if (testNb==0) result = basicTests(seed, ((double)proba) / 100); if (result) return 1; - return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); + return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration); } diff --git a/programs/fullbench.c b/programs/fullbench.c index 0c6e05e..0d08a40 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -30,7 +30,7 @@ #define _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -// Unix Large Files support (>4GB) +/* Unix Large Files support (>4GB) */ #if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions # define _LARGEFILE_SOURCE # define _FILE_OFFSET_BITS 64 @@ -47,17 +47,17 @@ /************************************** * Includes **************************************/ -#include <stdlib.h> // malloc -#include <stdio.h> // fprintf, fopen, ftello64 -#include <sys/types.h> // stat64 -#include <sys/stat.h> // stat64 -#include <string.h> // strcmp +#include <stdlib.h> /* malloc, free */ +#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <sys/types.h> /* stat64 */ +#include <sys/stat.h> /* stat64 */ +#include <string.h> /* strcmp */ -// Use ftime() if gettimeofday() is not available on your target +/* Use ftime() if gettimeofday() is not available on your target */ #if defined(BMK_LEGACY_TIMER) -# include <sys/timeb.h> // timeb, ftime +# include <sys/timeb.h> /* timeb, ftime */ #else -# include <sys/time.h> // gettimeofday +# include <sys/time.h> /* gettimeofday */ #endif #include "lz4.h" @@ -75,16 +75,11 @@ # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) #endif -// GCC does not support _rotl outside of Windows -#if !defined(_WIN32) -# define _rotl(x,r) ((x << r) | (x >> (32 - r))) -#endif - /************************************** * Basic Types **************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; @@ -118,7 +113,7 @@ #define GB *(1U<<30) #define KNUTH 2654435761U -#define MAX_MEM (1984 MB) +#define MAX_MEM (1920 MB) #define DEFAULT_CHUNKSIZE (4 MB) #define ALL_COMPRESSORS 0 @@ -139,53 +134,53 @@ struct chunkParameters /************************************** -* MACRO +* Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define PROGRESS(...) no_prompt ? 0 : DISPLAY(__VA_ARGS__) - +#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__) -//************************************** -// Benchmark Parameters -//************************************** -static int chunkSize = DEFAULT_CHUNKSIZE; -static int nbIterations = NBLOOPS; -static int BMK_pause = 0; -static int compressionTest = 1; -static int decompressionTest = 1; -static int compressionAlgo = ALL_COMPRESSORS; -static int decompressionAlgo = ALL_DECOMPRESSORS; -static int no_prompt = 0; +/************************************** +* Benchmark Parameters +**************************************/ +static int g_chunkSize = DEFAULT_CHUNKSIZE; +static int g_nbIterations = NBLOOPS; +static int g_pause = 0; +static int g_compressionTest = 1; +static int g_compressionAlgo = ALL_COMPRESSORS; +static int g_decompressionTest = 1; +static int g_decompressionAlgo = ALL_DECOMPRESSORS; +static int g_noPrompt = 0; -void BMK_SetBlocksize(int bsize) +static void BMK_setBlocksize(int bsize) { - chunkSize = bsize; - DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10); + g_chunkSize = bsize; + DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10); } -void BMK_SetNbIterations(int nbLoops) +static void BMK_setNbIterations(int nbLoops) { - nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", nbIterations); + g_nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", g_nbIterations); } -void BMK_SetPause(void) +static void BMK_setPause(void) { - BMK_pause = 1; + g_pause = 1; } -//********************************************************* -// Private functions -//********************************************************* + +/********************************************************* +* Private functions +*********************************************************/ #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) { - // Based on Legacy ftime() - // Rolls over every ~ 12.1 days (0x100000/24/60/60) - // Use GetMilliSpan to correct for rollover + /* Based on Legacy ftime() + * Rolls over every ~ 12.1 days (0x100000/24/60/60) + * Use GetMilliSpan to correct for rollover */ struct timeb tb; int nCount; ftime( &tb ); @@ -197,8 +192,8 @@ static int BMK_GetMilliStart(void) static int BMK_GetMilliStart(void) { - // Based on newer gettimeofday() - // Use GetMilliSpan to correct for rollover + /* Based on newer gettimeofday() + * Use GetMilliSpan to correct for rollover */ struct timeval tv; int nCount; gettimeofday(&tv, NULL); @@ -253,7 +248,7 @@ static U64 BMK_GetFileSize(char* infilename) struct stat statbuf; r = stat(infilename, &statbuf); #endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ return (U64)statbuf.st_size; } @@ -381,106 +376,141 @@ start: #endif // __SSSE3__ -static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) +static LZ4_stream_t LZ4_stream; +static void local_LZ4_resetDictT(void) { - return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); + LZ4_resetStream(&LZ4_stream); } -static void* stateLZ4; -static int local_LZ4_compress_withState(const char* in, char* out, int inSize) +static void local_LZ4_createStream(void) { - return LZ4_compress_withState(stateLZ4, in, out, inSize); + LZ4_resetStream(&LZ4_stream); } -static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) +static int local_LZ4_saveDict(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)); + (void)in; + return LZ4_saveDict(&LZ4_stream, out, inSize); } -static LZ4_stream_t* ctx; -static int local_LZ4_compress_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) { - return LZ4_compress_continue(ctx, in, out, inSize); + return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); } -static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_default_large(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)); } +static int local_LZ4_compress_default_small(const char* in, char* out, int inSize) +{ + return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1); +} -LZ4_stream_t LZ4_dict; -static void* local_LZ4_resetDictT(const char* fake) +static int local_LZ4_compress_fast0(const char* in, char* out, int inSize) { - (void)fake; - memset(&LZ4_dict, 0, sizeof(LZ4_stream_t)); - return NULL; + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0); } -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize); -static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast1(const char* in, char* out, int inSize) { - return LZ4_compress_forceExtDict(&LZ4_dict, in, out, inSize); + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1); } +static int local_LZ4_compress_fast2(const char* in, char* out, int inSize) +{ + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2); +} -static void* stateLZ4HC; -static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast17(const char* in, char* out, int inSize) { - return LZ4_compressHC_withStateHC(stateLZ4HC, in, out, inSize); + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17); } -static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); } -static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) +static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0); } -static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_withState(const char* in, char* out, int inSize) { - return LZ4_compressHC_continue((LZ4_streamHC_t*)ctx, in, out, inSize); + return LZ4_compress_withState(&LZ4_stream, in, out, inSize); } -static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) +static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_limitedOutput_withState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); } -static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) +static int local_LZ4_compress_continue(const char* in, char* out, int inSize) { - return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL); + return LZ4_compress_continue(&LZ4_stream, in, out, inSize); } -static int local_LZ4_saveDict(const char* in, char* out, int inSize) +static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) { - (void)in; - return LZ4_saveDict(&LZ4_dict, out, inSize); + return LZ4_compress_limitedOutput_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize)-1); +} + +/* declare hidden function */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize); + +static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize) +{ + return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize); +} + + +/* HC compression functions */ +LZ4_streamHC_t LZ4_streamHC; +static void local_LZ4_resetStreamHC(void) +{ + LZ4_resetStreamHC(&LZ4_streamHC, 0); } -LZ4_streamHC_t LZ4_dictHC; static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) { (void)in; - return LZ4_saveDictHC(&LZ4_dictHC, out, inSize); + return LZ4_saveDictHC(&LZ4_streamHC, out, inSize); } +static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_withStateHC(&LZ4_streamHC, in, out, inSize); +} -static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) +static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) { - (void)inSize; - //lz4_decode_sse((BYTE*)out, (BYTE*)in, inSize); - LZ4_decompress_fast(in, out, outSize); - return outSize; + return LZ4_compressHC_limitedOutput_withStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); +} + +static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); +} + +static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_continue(&LZ4_streamHC, in, out, inSize); } -static int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize) +static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize)-1); +} + + +/* decompression functions */ +static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) { (void)inSize; - LZ4_decompress_fast_withPrefix64k(in, out, outSize); + LZ4_decompress_fast(in, out, outSize); return outSize; } @@ -512,6 +542,13 @@ static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSi return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); } + +/* frame functions */ +static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) +{ + return (int)LZ4F_compressFrame(out, 2*inSize + 16, in, inSize, NULL); +} + static LZ4F_decompressionContext_t g_dCtx; static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize) @@ -526,83 +563,65 @@ static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outS } +#define NB_COMPRESSION_ALGORITHMS 100 +#define NB_DECOMPRESSION_ALGORITHMS 100 int fullSpeedBench(char** fileNamesTable, int nbFiles) { int fileIdx=0; - char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 16 - double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0}; - double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0}; -# define NB_DECOMPRESSION_ALGORITHMS 9 - double totalDTime[NB_DECOMPRESSION_ALGORITHMS+1] = {0}; size_t errorCode; + /* Init */ errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) - { - DISPLAY("dctx allocation issue \n"); - return 10; - } + if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } - // Loop for each file + /* Loop for each fileName */ while (fileIdx<nbFiles) { FILE* inFile; + char* orig_buff = NULL; + struct chunkParameters* chunkP = NULL; + char* compressed_buff=NULL; char* inFileName; U64 inFileSize; size_t benchedSize; int nbChunks; int maxCompressedChunkSize; - struct chunkParameters* chunkP; size_t readSize; - char* compressed_buff; int compressedBuffSize; + int compressedBuffSize; U32 crcOriginal; - - // Init - stateLZ4 = LZ4_createStream(); - stateLZ4HC = LZ4_createStreamHC(); - - // Check file existence + /* Check file existence */ inFileName = fileNamesTable[fileIdx++]; inFile = fopen( inFileName, "rb" ); - if (inFile==NULL) - { - DISPLAY( "Pb opening %s\n", inFileName); - return 11; - } + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - // Memory allocation & restrictions + /* Memory size adjustments */ inFileSize = BMK_GetFileSize(inFileName); - if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; } - benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2; - if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; } + if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; } + benchedSize = (size_t) BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ + if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) - { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } - // Alloc - chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*) malloc((size_t)benchedSize); - nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize); - maxCompressedChunkSize = LZ4_compressBound(chunkSize); + /* Allocation */ + chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters)); + orig_buff = (char*) malloc(benchedSize); + nbChunks = (int) ((benchedSize + (g_chunkSize-1)) / g_chunkSize); + maxCompressedChunkSize = LZ4_compressBound(g_chunkSize); compressedBuffSize = nbChunks * maxCompressedChunkSize; compressed_buff = (char*)malloc((size_t)compressedBuffSize); - - - if(!orig_buff || !compressed_buff) + if(!chunkP || !orig_buff || !compressed_buff) { - DISPLAY("\nError: not enough memory!\n"); - free(orig_buff); - free(compressed_buff); - free(chunkP); - fclose(inFile); - return 12; + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return(12); } - // Fill input buffer + /* Fill in src buffer */ DISPLAY("Loading %s... \r", inFileName); readSize = fread(orig_buff, 1, benchedSize, inFile); fclose(inFile); @@ -616,11 +635,11 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) return 13; } - // Calculating input Checksum - crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0); + /* Calculating input Checksum */ + crcOriginal = XXH32(orig_buff, benchedSize,0); - // Bench + /* Bench */ { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; size_t cSize=0; @@ -629,67 +648,80 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) DISPLAY("\r%79s\r", ""); DISPLAY(" %s : \n", inFileName); - // Compression Algorithms - for (cAlgNb=1; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) + /* Bench Compression Algorithms */ + for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) { const char* compressorName; int (*compressionFunction)(const char*, char*, int); - void* (*initFunction)(const char*) = NULL; + void (*initFunction)(void) = NULL; double bestTime = 100000000.; - // Init data chunks + /* filter compressionAlgo only */ + if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue; + + /* Init data chunks */ { int i; size_t remaining = benchedSize; char* in = orig_buff; char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize); + nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); for (i=0; i<nbChunks; i++) { chunkP[i].id = i; - chunkP[i].origBuffer = in; in += chunkSize; - if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].origBuffer = in; in += g_chunkSize; + if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } } - if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; - switch(cAlgNb) { - case 1 : compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; - case 2 : compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break; - case 3 : compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break; - case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; - case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break; - case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break; - case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; - case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; - case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; - case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; - case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; - case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; - case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; - case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; + case 0 : DISPLAY("Compression functions : \n"); continue; + case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break; + case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break; + case 3 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break; + case 4 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break; + case 5 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break; + case 6 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break; + case 7 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break; + case 8 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break; + + case 10: compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; + case 11: compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; + case 12 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; + case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; + case 14: compressionFunction = local_LZ4_compressHC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_continue"; break; + case 15: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; + case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; + case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; chunkP[0].origSize = (int)benchedSize; nbChunks=1; break; - case 15: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; - LZ4_loadDict(&LZ4_dict, chunkP[0].origBuffer, chunkP[0].origSize); + case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; + LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize); break; - case 16: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; - LZ4_loadDictHC(&LZ4_dictHC, chunkP[0].origBuffer, chunkP[0].origSize); + case 41: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; + LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize); break; - default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; + case 60: DISPLAY("Obsolete compression functions : \n"); continue; + case 61: compressionFunction = LZ4_compress; compressorName = "LZ4_compress"; break; + case 62: compressionFunction = local_LZ4_compress_limitedOutput; compressorName = "LZ4_compress_limitedOutput"; break; + case 63: compressionFunction = local_LZ4_compress_withState; compressorName = "LZ4_compress_withState"; break; + case 64: compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; + case 65: compressionFunction = local_LZ4_compress_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_continue"; break; + case 66: compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_limitedOutput_continue"; break; + default : + continue; /* unknown ID : just skip */ } - for (loopNb = 1; loopNb <= nbIterations; loopNb++) + for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { double averageTime; int milliTime; PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); - { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warming up memory + { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */ nb_loops = 0; milliTime = BMK_GetMilliStart(); @@ -697,17 +729,17 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) milliTime = BMK_GetMilliStart(); while(BMK_GetMilliSpan(milliTime) < TIMELOOP) { - if (initFunction!=NULL) ctx = (LZ4_stream_t*)initFunction(chunkP[0].origBuffer); + if (initFunction!=NULL) initFunction(); for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize); if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", compressorName), exit(1); } - if (initFunction!=NULL) free(ctx); nb_loops++; } milliTime = BMK_GetMilliSpan(milliTime); + nb_loops += !nb_loops; /* avoid division by zero */ averageTime = (double)milliTime / nb_loops; if (averageTime < bestTime) bestTime = averageTime; cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize; @@ -719,24 +751,22 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); else DISPLAY("%2i-%-28.28s :%9i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); - - totalCTime[cAlgNb] += bestTime; - totalCSize[cAlgNb] += cSize; } - // Prepare layout for decompression - // Init data chunks + /* Prepare layout for decompression */ + /* Init data chunks */ { int i; size_t remaining = benchedSize; char* in = orig_buff; char* out = compressed_buff; - nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize); + + nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize); for (i=0; i<nbChunks; i++) { chunkP[i].id = i; - chunkP[i].origBuffer = in; in += chunkSize; - if ((int)remaining > chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].origBuffer = in; in += g_chunkSize; + if ((int)remaining > g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; chunkP[i].compressedSize = 0; } @@ -747,39 +777,45 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); } - // Decompression Algorithms - for (dAlgNb=1; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++) + /* Decompression Algorithms */ + for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (g_decompressionTest); dAlgNb++) { - //const char* dName = decompressionNames[dAlgNb]; const char* dName; int (*decompressionFunction)(const char*, char*, int, int); double bestTime = 100000000.; - if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != dAlgNb)) continue; + if ((g_decompressionAlgo != ALL_DECOMPRESSORS) && (g_decompressionAlgo != dAlgNb)) continue; switch(dAlgNb) { + case 0: DISPLAY("Decompression functions : \n"); continue; case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break; - case 2: decompressionFunction = local_LZ4_decompress_fast_withPrefix64k; dName = "LZ4_decompress_fast_withPrefix64k"; break; case 3: decompressionFunction = local_LZ4_decompress_fast_usingDict; dName = "LZ4_decompress_fast_usingDict"; break; case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break; - case 5: decompressionFunction = LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break; case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break; case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; break; case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break; case 9: decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; errorCode = LZ4F_compressFrame(compressed_buff, compressedBuffSize, orig_buff, benchedSize, NULL); - if (LZ4F_isError(errorCode)) { DISPLAY("Preparation error compressing frame\n"); return 1; } + if (LZ4F_isError(errorCode)) + { + DISPLAY("Error while preparing compressed frame\n"); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return 1; + } chunkP[0].origSize = (int)benchedSize; chunkP[0].compressedSize = (int)errorCode; nbChunks = 1; break; - default : DISPLAY("ERROR ! Bad decompression algorithm Id !! \n"); free(chunkP); return 1; + default : + continue; /* skip if unknown ID */ } - { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } // zeroing source area, for CRC checking + { size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */ - for (loopNb = 1; loopNb <= nbIterations; loopNb++) + for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { double averageTime; int milliTime; @@ -802,6 +838,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) } milliTime = BMK_GetMilliSpan(milliTime); + nb_loops += !nb_loops; /* Avoid division by zero */ averageTime = (double)milliTime / nb_loops; if (averageTime < bestTime) bestTime = averageTime; @@ -813,24 +850,21 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) } DISPLAY("%2i-%-29.29s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); - - totalDTime[dAlgNb] += bestTime; } - } - free(orig_buff); free(compressed_buff); free(chunkP); } - if (BMK_pause) { printf("press enter...\n"); getchar(); } + LZ4F_freeDecompressionContext(g_dCtx); + if (g_pause) { printf("press enter...\n"); (void)getchar(); } return 0; } -int usage(char* exename) +static int usage(char* exename) { DISPLAY( "Usage :\n"); DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); @@ -841,7 +875,7 @@ int usage(char* exename) return 0; } -int usage_advanced(void) +static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS); @@ -851,7 +885,7 @@ int usage_advanced(void) return 0; } -int badusage(char* exename) +static int badusage(char* exename) { DISPLAY("Wrong parameters\n"); usage(exename); @@ -877,7 +911,7 @@ int main(int argc, char** argv) if(!argument) continue; // Protection if argument empty if (!strcmp(argument, "--no-prompt")) { - no_prompt = 1; + g_noPrompt = 1; continue; } @@ -892,22 +926,22 @@ int main(int argc, char** argv) { // Select compression algorithm only case 'c': - decompressionTest = 0; + g_decompressionTest = 0; while ((argument[1]>= '0') && (argument[1]<= '9')) { - compressionAlgo *= 10; - compressionAlgo += argument[1] - '0'; + g_compressionAlgo *= 10; + g_compressionAlgo += argument[1] - '0'; argument++; } break; // Select decompression algorithm only case 'd': - compressionTest = 0; + g_compressionTest = 0; while ((argument[1]>= '0') && (argument[1]<= '9')) { - decompressionAlgo *= 10; - decompressionAlgo += argument[1] - '0'; + g_decompressionAlgo *= 10; + g_decompressionAlgo += argument[1] - '0'; argument++; } break; @@ -928,7 +962,7 @@ int main(int argc, char** argv) { int B = argument[1] - '0'; int S = 1 << (8 + 2*B); - BMK_SetBlocksize(S); + BMK_setBlocksize(S); argument++; break; } @@ -940,16 +974,16 @@ _exit_blockProperties: // Modify Nb Iterations case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) + if ((argument[1] >='0') && (argument[1] <='9')) { int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); + BMK_setNbIterations(iters); argument++; } break; // Pause at the end (hidden option) - case 'p': BMK_SetPause(); break; + case 'p': BMK_setPause(); break; // Unknown command default : badusage(exename); return 1; diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 3d3cf8e..6588290 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -19,7 +19,6 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4 - LZ4 source mirror : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ @@ -154,10 +153,10 @@ static U32 FUZ_rand(U32* src) { U32 rand32 = *src; rand32 *= PRIME1; - rand32 += PRIME2; + rand32 ^= PRIME2; rand32 = FUZ_rotl32(rand32, 13); *src = rand32; - return rand32 >> 3; + return rand32; } @@ -169,27 +168,28 @@ static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, dou size_t pos = 0; U32 P32 = (U32)(32768 * proba); - // First Byte - BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); + /* First Bytes */ + while (pos < 20) + BBuffer[pos++] = (BYTE)(FUZ_rand(seed)); while (pos < bufferSize) { - // Select : Literal (noise) or copy (within 64K) + /* Select : Literal (noise) or copy (within 64K) */ if (FUZ_RAND15BITS < P32) { - // Copy (within 64K) + /* Copy (within 64K) */ size_t match, d; size_t length = FUZ_RANDLENGTH + 4; size_t offset = FUZ_RAND15BITS + 1; - if (offset > pos) offset = pos; + while (offset > pos) offset >>= 1; d = pos + length; - if (d > bufferSize) d = bufferSize; + while (d > bufferSize) d = bufferSize; match = pos - offset; while (pos < d) BBuffer[pos++] = BBuffer[match++]; } else { - // Literal (noise) + /* Literal (noise) */ size_t d; size_t length = FUZ_RANDLENGTH; d = pos + length; @@ -210,7 +210,7 @@ static int FUZ_AddressOverflow(void) printf("Overflow tests : "); - // Only possible in 32-bits + /* Only possible in 32-bits */ if (sizeof(void*)==8) { printf("64 bits mode : no overflow \n"); @@ -300,7 +300,17 @@ static void FUZ_displayUpdate(unsigned testNb) } -static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const double compressibility) +static void FUZ_findDiff(const void* buff1, const void* buff2) +{ + const BYTE* b1 = (const BYTE*)buff1; + const BYTE* b2 = (const BYTE*)buff2; + size_t i=0; + while (b1[i]==b2[i]) i++; + DISPLAY("Wrong Byte at position %u\n", (unsigned)i); +} + + +static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration) { unsigned long long bytes = 0; unsigned long long cbytes = 0; @@ -313,34 +323,35 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do int ret; unsigned cycleNb; # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \ - printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } + printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; } # define FUZ_DISPLAYTEST { testNb++; g_displayLevel<3 ? 0 : printf("%2u\b\b", testNb); if (g_displayLevel==4) fflush(stdout); } void* stateLZ4 = malloc(LZ4_sizeofState()); void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); - void* LZ4continue; LZ4_stream_t LZ4dict; LZ4_streamHC_t LZ4dictHC; U32 crcOrig, crcCheck; U32 coreRandState = seed; U32 randState = coreRandState ^ PRIME3; int result = 0; + const U32 startTime = FUZ_GetMilliStart(); - // init + /* init */ memset(&LZ4dict, 0, sizeof(LZ4dict)); + duration *= 1000; - // Create compressible test buffer + /* Create compressible test buffer */ CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH); FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState); compressedBuffer = (char*)malloc(LZ4_compressBound(FUZ_MAX_BLOCK_SIZE)); decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE); - // move to startCycle + /* move to startCycle */ for (cycleNb = 0; cycleNb < startCycle; cycleNb++) { (void)FUZ_rand(&coreRandState); - if (0) // some problems related to dictionary re-use; in this case, enable this loop + if (0) /* some problems are related to dictionary re-use; in this case, enable this loop */ { int dictSize, blockSize, blockStart; char* dict; @@ -362,8 +373,8 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do } } - // Test loop - for (cycleNb = startCycle; cycleNb < nbCycles; cycleNb++) + /* Main test loop */ + for (cycleNb = startCycle; (cycleNb < nbCycles) || (FUZ_GetMilliSpan(startTime) < duration) ; cycleNb++) { U32 testNb = 0; char* dict; @@ -375,7 +386,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do (void)FUZ_rand(&coreRandState); randState = coreRandState ^ PRIME3; - // Select block to test + /* Select block to test */ blockSize = FUZ_rand(&randState) % FUZ_MAX_BLOCK_SIZE; blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize); dictSize = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE; @@ -385,23 +396,58 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do /* Compression tests */ - // Test compression HC + /* Test compression destSize */ + FUZ_DISPLAYTEST; + { + int srcSize = blockSize; + int targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7; + char endCheck = FUZ_rand(&randState) & 255; + compressedBuffer[targetSize] = endCheck; + ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize); + FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !"); + FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !"); + FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !"); + DISPLAY("destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize); + if (targetSize>0) + { + FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed"); + /* check correctness */ + FUZ_DISPLAYTEST; + + crcOrig = XXH32(block, srcSize, 0); + compressedSize = ret; + endCheck = FUZ_rand(&randState) & 255; + decodedBuffer[srcSize] = endCheck; + ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize"); + FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data"); + FUZ_CHECKTEST(decodedBuffer[srcSize] != endCheck, "LZ4_decompress_safe() overwrite dst buffer !"); + crcCheck = XXH32(decodedBuffer, srcSize, 0); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe() corrupted decoded data"); + + DISPLAY(" OK \n"); + } + else + DISPLAY(" \n"); + } + + /* Test compression HC */ FUZ_DISPLAYTEST; ret = LZ4_compressHC(block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed"); HCcompressedSize = ret; - // Test compression HC using external state + /* Test compression HC using external state */ FUZ_DISPLAYTEST; ret = LZ4_compressHC_withStateHC(stateLZ4HC, block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); - // Test compression using external state + /* Test compression using external state */ FUZ_DISPLAYTEST; ret = LZ4_compress_withState(stateLZ4, block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); - // Test compression + /* Test compression */ FUZ_DISPLAYTEST; ret = LZ4_compress(block, compressedBuffer, blockSize); FUZ_CHECKTEST(ret==0, "LZ4_compress() failed"); @@ -411,7 +457,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do crcOrig = XXH32(block, blockSize, 0); - // Test decoding with output size being exactly what's necessary => must work + /* Test decoding with output size being exactly what's necessary => must work */ FUZ_DISPLAYTEST; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize); FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); @@ -419,19 +465,19 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do crcCheck = XXH32(decodedBuffer, blockSize, 0); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data"); - // Test decoding with one byte missing => must fail + /* Test decoding with one byte missing => must fail */ FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer"); - // Test decoding with one byte too much => must fail + /* Test decoding with one byte too much => must fail */ FUZ_DISPLAYTEST; ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1); FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); - // Test decoding with output size exactly what's necessary => must work + /* Test decoding with output size exactly what's necessary => must work */ FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize); @@ -545,16 +591,18 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do /* Compress using dictionary */ FUZ_DISPLAYTEST; - LZ4continue = LZ4_create (dict); - LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, dict, compressedBuffer, dictSize); // Just to fill hash tables - blockContinueCompressedSize = LZ4_compress_continue ((LZ4_stream_t*)LZ4continue, block, compressedBuffer, blockSize); - FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); - free (LZ4continue); + { + LZ4_stream_t LZ4_stream; + LZ4_resetStream(&LZ4_stream); + LZ4_compress_continue (&LZ4_stream, dict, compressedBuffer, dictSize); /* Just to fill hash tables */ + blockContinueCompressedSize = LZ4_compress_continue (&LZ4_stream, block, compressedBuffer, blockSize); + FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_continue failed"); + } /* Decompress with dictionary as prefix */ FUZ_DISPLAYTEST; memcpy(decodedBuffer, dict, dictSize); - ret = LZ4_decompress_fast_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockSize); + ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize); FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_withPrefix64k did not read all compressed block input"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); if (crcCheck!=crcOrig) @@ -567,14 +615,14 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_withPrefix64k corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_withPrefix64k(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize); - FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_withPrefix64k did not regenerate original data"); + ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize); + FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data"); crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0); - FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_withPrefix64k corrupted decoded data"); + FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - // Compress using External dictionary + /* Compress using External dictionary */ FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 0xF) + 1; // Separation, so it is an ExtDict + dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_loadDict(&LZ4dict, dict, dictSize); blockContinueCompressedSize = LZ4_compress_continue(&LZ4dict, block, compressedBuffer, blockSize); @@ -583,7 +631,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); ret = LZ4_compress_limitedOutput_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); - FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); + FUZ_CHECKTEST(ret>0, "LZ4_compress_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize); FUZ_DISPLAYTEST; LZ4_loadDict(&LZ4dict, dict, dictSize); @@ -591,7 +639,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize); FUZ_CHECKTEST(ret<=0, "LZ4_compress_limitedOutput_continue should work : enough size available within output buffer"); - // Decompress with dictionary as external + /* Decompress with dictionary as external */ FUZ_DISPLAYTEST; decodedBuffer[blockSize] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize); @@ -599,11 +647,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size") crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) - { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - } + FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize); FUZ_DISPLAYTEST; @@ -617,7 +661,7 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_DISPLAYTEST; decodedBuffer[blockSize-1] = 0; ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_withDict should have failed : wrong original size (-1 byte)"); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)"); FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size"); FUZ_DISPLAYTEST; @@ -638,9 +682,9 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do } } - // Compress HC using External dictionary + /* Compress HC using External dictionary */ FUZ_DISPLAYTEST; - dict -= (FUZ_rand(&randState) & 7); // even bigger separation + dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */ if (dict < (char*)CNBuffer) dict = (char*)CNBuffer; LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); @@ -665,29 +709,26 @@ static int FUZ_test(U32 seed, const U32 nbCycles, const U32 startCycle, const do FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size") crcCheck = XXH32(decodedBuffer, blockSize, 0); if (crcCheck!=crcOrig) - { - int i=0; - while (block[i]==decodedBuffer[i]) i++; - printf("Wrong Byte at position %i/%i\n", i, blockSize); - } + FUZ_findDiff(block, decodedBuffer); FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data"); - - // ***** End of tests *** // - // Fill stats + /* ***** End of tests *** */ + /* Fill stats */ bytes += blockSize; cbytes += compressedSize; hcbytes += HCcompressedSize; ccbytes += blockContinueCompressedSize; } + if (nbCycles<=1) nbCycles = cycleNb; /* end by time */ + bytes += !bytes; /* avoid division by 0 */ printf("\r%7u /%7u - ", cycleNb, nbCycles); printf("all tests completed successfully \n"); printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100); - // unalloc + /* release memory */ { _exit: free(CNBuffer); @@ -719,13 +760,13 @@ static void FUZ_unitTests(void) char ringBuffer[ringBufferSize]; U32 randState = 1; - // Init + /* Init */ FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState); - // 32-bits address space overflow test + /* 32-bits address space overflow test */ FUZ_AddressOverflow(); - // LZ4 streaming tests + /* LZ4 streaming tests */ { LZ4_stream_t* statePtr; LZ4_stream_t streamingState; @@ -733,12 +774,12 @@ static void FUZ_unitTests(void) U64 crcNew; int result; - // Allocation test + /* Allocation test */ statePtr = LZ4_createStream(); FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed"); LZ4_freeStream(statePtr); - // simple compression test + /* simple compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); LZ4_resetStream(&streamingState); result = LZ4_compress_limitedOutput_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1); @@ -749,7 +790,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - // ring buffer test + /* ring buffer test */ { XXH64_state_t xxhOrig; XXH64_state_t xxhNew; @@ -783,7 +824,7 @@ static void FUZ_unitTests(void) crcNew = XXH64_digest(&xxhNew); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - // prepare next message + /* prepare next message */ iNext += messageSize; rNext += messageSize; dNext += messageSize; @@ -794,21 +835,20 @@ static void FUZ_unitTests(void) } } - // LZ4 HC streaming tests + /* LZ4 HC streaming tests */ { LZ4_streamHC_t* sp; LZ4_streamHC_t sHC; - //XXH64_state_t xxh; U64 crcOrig; U64 crcNew; int result; - // Allocation test + /* Allocation test */ sp = LZ4_createStreamHC(); FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed"); LZ4_freeStreamHC(sp); - // simple compression test + /* simple HC compression test */ crcOrig = XXH64(testInput, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1); @@ -819,7 +859,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); - // simple dictionary compression test + /* simple dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, 64 KB); @@ -831,7 +871,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); - // multiple HC compression test with dictionary + /* multiple HC compression test with dictionary */ { int result1, result2; int segSize = testCompressedSize / 2; @@ -851,7 +891,7 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); } - // remote dictionary HC compression test + /* remote dictionary HC compression test */ crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_loadDictHC(&sHC, testInput, 32 KB); @@ -863,7 +903,7 @@ static void FUZ_unitTests(void) crcNew = XXH64(testVerify, testCompressedSize, 0); FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); - // multiple HC compression with ext. dictionary + /* multiple HC compression with ext. dictionary */ { XXH64_state_t crcOrigState; XXH64_state_t crcNewState; @@ -912,7 +952,7 @@ static void FUZ_unitTests(void) } } - // ring buffer test + /* ring buffer test */ { XXH64_state_t xxhOrig; XXH64_state_t xxhNew; @@ -956,24 +996,51 @@ static void FUZ_unitTests(void) } } - // small decoder-side ring buffer test + /* small decoder-side ring buffer test */ { XXH64_state_t xxhOrig; XXH64_state_t xxhNew; LZ4_streamDecode_t decodeState; - const U32 maxMessageSizeLog = 10; + const U32 maxMessageSizeLog = 12; const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1; - U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; + U32 messageSize; U32 totalMessageSize = 0; U32 iNext = 0; U32 dNext = 0; - const U32 dBufferSize = 64 KB + maxMessageSizeMask; + const U32 dBufferSize = 64 KB; XXH64_reset(&xxhOrig, 0); XXH64_reset(&xxhNew, 0); LZ4_resetStreamHC(&sHC, 0); LZ4_setStreamDecode(&decodeState, NULL, 0); +#define BSIZE1 65537 +#define BSIZE2 16435 + + /* first block */ + + messageSize = BSIZE1; + XXH64_update(&xxhOrig, testInput + iNext, messageSize); + crcOrig = XXH64_digest(&xxhOrig); + + result = LZ4_compressHC_limitedOutput_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize); + FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); + + result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); + FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); + + XXH64_update(&xxhNew, testVerify + dNext, messageSize); + crcNew = XXH64_digest(&xxhNew); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + + /* prepare next message */ + dNext += messageSize; + totalMessageSize += messageSize; + messageSize = BSIZE2; + iNext = 132000; + memcpy(testInput + iNext, testInput + 8, messageSize); + if (dNext > dBufferSize) dNext = 0; + while (totalMessageSize < 9 MB) { XXH64_update(&xxhOrig, testInput + iNext, messageSize); @@ -983,18 +1050,20 @@ static void FUZ_unitTests(void) FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed"); result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize); - FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed"); + FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed"); XXH64_update(&xxhNew, testVerify + dNext, messageSize); crcNew = XXH64_digest(&xxhNew); - FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); + if (crcOrig != crcNew) + FUZ_findDiff(testInput + iNext, testVerify + dNext); + FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test"); /* prepare next message */ dNext += messageSize; totalMessageSize += messageSize; messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1; iNext = (FUZ_rand(&randState) & 65535); - if (dNext + messageSize > dBufferSize) dNext = 0; + if (dNext > dBufferSize) dNext = 0; } } } @@ -1013,6 +1082,7 @@ static int FUZ_usage(char* programName) DISPLAY( "\n"); DISPLAY( "Arguments :\n"); DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS); + DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n"); DISPLAY( " -s# : Select seed (default:prompt user)\n"); DISPLAY( " -t# : Select starting test number (default:0)\n"); DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); @@ -1033,8 +1103,9 @@ int main(int argc, char** argv) int proba = FUZ_COMPRESSIBILITY_DEFAULT; int pause = 0; char* programName = argv[0]; + U32 duration = 0; - // Check command line + /* Check command line */ for(argNb=1; argNb<argc; argNb++) { char* argument = argv[argNb]; @@ -1066,7 +1137,7 @@ int main(int argc, char** argv) case 'i': argument++; - nbTests=0; + nbTests = 0; duration = 0; while ((*argument>='0') && (*argument<='9')) { nbTests *= 10; @@ -1075,6 +1146,31 @@ int main(int argc, char** argv) } break; + case 'T': + argument++; + nbTests = 0; duration = 0; + for (;;) + { + switch(*argument) + { + case 'm': duration *= 60; argument++; continue; + case 's': + case 'n': argument++; continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': duration *= 10; duration += *argument++ - '0'; continue; + } + break; + } + break; + case 's': argument++; seed=0; seedset=1; @@ -1115,7 +1211,6 @@ int main(int argc, char** argv) } } - // Get Seed printf("Starting LZ4 fuzzer (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION); if (!seedset) seed = FUZ_GetMilliStart() % 10000; @@ -1127,11 +1222,11 @@ int main(int argc, char** argv) if (nbTests<=0) nbTests=1; { - int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100); + int result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration); if (pause) { DISPLAY("press enter ... \n"); - getchar(); + (void)getchar(); } return result; } diff --git a/programs/lz4.1 b/programs/lz4.1 index 1225832..a79fed3 100644 --- a/programs/lz4.1 +++ b/programs/lz4.1 @@ -158,13 +158,13 @@ for files that have not been compressed with force write to standard output, even if it is the console .TP -.BR \-m +.BR \-m ", " \--multiple Multiple file names. By default, the second filename is used as the output filename for the compressed file. With .B -m -, you can specify any number of input filenames, each of them will be compressed -with the resulting compressed file named +, you can specify any number of input filenames. Each of them will be compressed +independently, and the resulting name of the compressed file will be .B filename.lz4 . @@ -177,15 +177,15 @@ with the resulting compressed file named block dependency (improve compression ratio) .TP .B \--[no-]frame-crc - disable stream checksum (default:enabled) + select frame checksum (default:enabled) .TP .B \--[no-]content-size - compressed file includes original size (default:not present) + header includes original size (default:not present) Note : this option can only be activated when the original size can be determined, -hence for a file. It won't work with unknown source size, such as stdin pipe. +hence for a file. It won't work with unknown source size, such as stdin or pipe. .TP .B \--[no-]sparse - enable sparse file (default:disabled)(experimental) + sparse file support (default:enabled) .TP .B \-l use Legacy format (useful for Linux Kernel compression) diff --git a/programs/lz4cli.c b/programs/lz4cli.c index da5da71..970856d 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -65,7 +65,7 @@ /**************************** * OS-specific Includes *****************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) # include <io.h> /* _isatty */ # ifdef __MINGW32__ int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */ @@ -102,7 +102,7 @@ ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static unsigned displayLevel = 2; /* 0 : no display ; 1: errors ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */ +static unsigned displayLevel = 2; /* 0 : no display ; 1: errors only ; 2 : downgradable normal ; 3 : non-downgradable normal; 4 : + information */ /************************************** @@ -175,7 +175,7 @@ static int usage_advanced(void) /* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */ DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled)\n"); DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n"); - DISPLAY( "--sparse : enable sparse file (default:disabled)(experimental)\n"); + DISPLAY( "--[no-]sparse : sparse file support (default:enabled)\n"); DISPLAY( "Benchmark arguments :\n"); DISPLAY( " -b : benchmark file(s)\n"); DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); @@ -251,7 +251,7 @@ static int badusage(void) static void waitEnter(void) { DISPLAY("Press enter to continue...\n"); - getchar(); + (void)getchar(); } @@ -265,7 +265,8 @@ int main(int argc, char** argv) forceStdout=0, forceCompress=0, main_pause=0, - multiple_inputs=0; + multiple_inputs=0, + operationResult=0; const char* input_filename=0; const char* output_filename=0; char* dynNameSpace=0; @@ -294,12 +295,13 @@ int main(int argc, char** argv) /* long commands (--long-word) */ if (!strcmp(argument, "--compress")) { forceCompress = 1; continue; } if ((!strcmp(argument, "--decompress")) - || (!strcmp(argument, "--uncompress"))) { decode = 1; continue; } + || (!strcmp(argument, "--uncompress"))) { decode = 1; continue; } + if (!strcmp(argument, "--multiple")) { multiple_inputs = 1; if (inFileNames==NULL) inFileNames = (const char**)malloc(argc * sizeof(char*)); continue; } if (!strcmp(argument, "--test")) { decode = 1; LZ4IO_setOverwrite(1); output_filename=nulmark; continue; } if (!strcmp(argument, "--force")) { LZ4IO_setOverwrite(1); continue; } if (!strcmp(argument, "--no-force")) { LZ4IO_setOverwrite(0); continue; } if ((!strcmp(argument, "--stdout")) - || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; } + || (!strcmp(argument, "--to-stdout"))) { forceStdout=1; output_filename=stdoutmark; displayLevel=1; continue; } if (!strcmp(argument, "--frame-crc")) { LZ4IO_setStreamChecksumMode(1); continue; } if (!strcmp(argument, "--no-frame-crc")) { LZ4IO_setStreamChecksumMode(0); continue; } if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(1); continue; } @@ -311,6 +313,7 @@ int main(int argc, char** argv) if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE); return 0; } if (!strcmp(argument, "--keep")) { continue; } /* keep source file (default anyway; just for xz/lzma compatibility) */ + /* Short commands (note : aggregated short commands are allowed) */ if (argument[0]=='-') { @@ -420,11 +423,15 @@ int main(int argc, char** argv) /* Modify Nb Iterations (benchmark only) */ case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) { - int iters = argument[1] - '0'; + unsigned iters = 0; + while ((argument[1] >='0') && (argument[1] <='9')) + { + iters *= 10; + iters += argument[1] - '0'; + argument++; + } BMK_setNbIterations(iters); - argument++; } break; @@ -463,14 +470,19 @@ int main(int argc, char** argv) if (!decode) DISPLAYLEVEL(4, "Blocks size : %i KB\n", blockSize>>10); /* No input filename ==> use stdin */ - if (multiple_inputs) input_filename = inFileNames[0], output_filename = (char*)(inFileNames[0]); + if (multiple_inputs) input_filename = inFileNames[0], output_filename = (const char*)(inFileNames[0]); if(!input_filename) { input_filename=stdinmark; } /* Check if input or output are defined as console; trigger an error in this case */ if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); /* Check if benchmark is selected */ - if (bench) return BMK_benchFiles(inFileNames, ifnIdx, cLevel); + if (bench) + { + int bmkResult = BMK_benchFiles(inFileNames, ifnIdx, cLevel); + free((void*)inFileNames); + return bmkResult; + } /* No output filename ==> try to select one automatically (when possible) */ while (!output_filename) @@ -509,32 +521,40 @@ int main(int argc, char** argv) /* Check if output is defined as console; trigger an error in this case */ if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); - /* No warning message in pure pipe mode (stdin + stdout) */ + /* Downgrade notification level in pure pipe mode (stdin + stdout) and multiple file mode */ if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; + if ((multiple_inputs) && (displayLevel==2)) displayLevel=1; /* IO Stream/File */ LZ4IO_setNotificationLevel(displayLevel); - if (decode) DEFAULT_DECOMPRESSOR(input_filename, output_filename); + if (decode) + { + if (multiple_inputs) + operationResult = LZ4IO_decompressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION); + else + DEFAULT_DECOMPRESSOR(input_filename, output_filename); + } else { - /* compression is default action */ - if (legacy_format) - { - DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); - LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); - } + /* compression is default action */ + if (legacy_format) + { + DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated) ! \n"); + LZ4IO_compressFilename_Legacy(input_filename, output_filename, cLevel); + } + else + { + if (multiple_inputs) + operationResult = LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); else - { - if (multiple_inputs) - LZ4IO_compressMultipleFilenames(inFileNames, ifnIdx, LZ4_EXTENSION, cLevel); - else - DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); - } + DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + } } if (main_pause) waitEnter(); free(dynNameSpace); free((void*)inFileNames); + if (operationResult != 0) return operationResult; return 0; } diff --git a/programs/lz4io.c b/programs/lz4io.c index 02e03c8..209f5ed 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -46,22 +46,22 @@ /***************************** * Includes *****************************/ -#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout */ +#include <stdio.h> /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */ #include <stdlib.h> /* malloc, free */ #include <string.h> /* strcmp, strlen */ #include <time.h> /* clock */ #include <sys/types.h> /* stat64 */ #include <sys/stat.h> /* stat64 */ #include "lz4io.h" -#include "lz4.h" /* still required for legacy format */ -#include "lz4hc.h" /* still required for legacy format */ +#include "lz4.h" /* still required for legacy format */ +#include "lz4hc.h" /* still required for legacy format */ #include "lz4frame.h" /****************************** * OS-specific Includes ******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) # include <fcntl.h> /* _O_BINARY */ # include <io.h> /* _setmode, _fileno, _get_osfhandle */ # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) @@ -131,7 +131,7 @@ static int g_blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT; static int g_blockChecksum = 0; static int g_streamChecksum = 1; static int g_blockIndependence = 1; -static int g_sparseFileSupport = 0; +static int g_sparseFileSupport = 1; static int g_contentSizeFlag = 0; static const int minBlockSizeID = 4; @@ -159,7 +159,7 @@ static const int maxBlockSizeID = 7; #define EXTENDED_ARGUMENTS #define EXTENDED_HELP #define EXTENDED_FORMAT -#define DEFAULT_DECOMPRESSOR decodeLZ4S +#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F /* ************************************************** */ @@ -240,7 +240,7 @@ static unsigned long long LZ4IO_GetFileSize(const char* infilename) struct stat statbuf; r = stat(infilename, &statbuf); #endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* failure, or is not a regular file */ return (unsigned long long)statbuf.st_size; } @@ -253,7 +253,7 @@ static int LZ4IO_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id)) static int LZ4IO_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0; } -static int get_fileHandle(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) +static int LZ4IO_getFiles(const char* input_filename, const char* output_filename, FILE** pfinput, FILE** pfoutput) { if (!strcmp (input_filename, stdinmark)) @@ -267,6 +267,12 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam *pfinput = fopen(input_filename, "rb"); } + if ( *pfinput==0 ) + { + DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", input_filename); + return 1; + } + if (!strcmp (output_filename, stdoutmark)) { DISPLAYLEVEL(4,"Using stdout for output\n"); @@ -283,18 +289,18 @@ static int get_fileHandle(const char* input_filename, const char* output_filenam fclose(*pfoutput); if (!g_overwrite) { - char ch; + int ch = 'Y'; DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); - DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); - if (g_displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ - ch = (char)getchar(); - if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); + if ((g_displayLevel <= 1) || (*pfinput == stdin)) + EXM_THROW(11, "Operation aborted : %s already exists", output_filename); /* No interaction possible */ + DISPLAYLEVEL(2, "Overwrite ? (Y/n) : "); + while((ch = getchar()) != '\n' && ch != EOF) /* flush integrated */ + if ((ch!='Y') && (ch!='y')) EXM_THROW(12, "No. Operation aborted : %s already exists", output_filename); } } *pfoutput = fopen( output_filename, "wb" ); } - if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); return 0; @@ -316,16 +322,23 @@ static void LZ4IO_writeLE32 (void* p, unsigned value32) dstPtr[3] = (unsigned char)(value32 >> 24); } +static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel) +{ + (void)cLevel; + return LZ4_compress_fast(src, dst, srcSize, dstSize, 1); +} + /* LZ4IO_compressFilename_Legacy : * This function is intentionally "hidden" (not published in .h) * It generates compressed streams using the old 'legacy' format */ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename, int compressionlevel) { - int (*compressionFunction)(const char*, char*, int); + int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel); unsigned long long filesize = 0; unsigned long long compressedfilesize = MAGICNUMBER_SIZE; char* in_buff; char* out_buff; + const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE); FILE* finput; FILE* foutput; clock_t start, end; @@ -334,14 +347,14 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /* Init */ start = clock(); - if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; + if (compressionlevel < 3) compressionFunction = LZ4IO_LZ4_compress; else compressionFunction = LZ4_compress_HC; - get_fileHandle(input_filename, output_filename, &finput, &foutput); - if ((g_displayLevel==2) && (compressionlevel==1)) g_displayLevel=3; + if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) + EXM_THROW(20, "File error"); /* Allocate Memory */ in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + out_buff = (char*)malloc(outBuffSize); if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); /* Write Archive Header */ @@ -359,9 +372,9 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output filesize += inSize; /* Compress Block */ - outSize = compressionFunction(in_buff, out_buff+4, inSize); + outSize = compressionFunction(in_buff, out_buff+4, inSize, outBuffSize, compressionlevel); compressedfilesize += outSize+4; - DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ LZ4IO_writeLE32(out_buff, outSize); @@ -372,6 +385,7 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output /* Status */ end = clock(); DISPLAYLEVEL(2, "\r%79s\r", ""); + filesize += !filesize; /* avoid divide by zero */ DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); { @@ -393,128 +407,210 @@ int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output * Compression using Frame format *********************************************/ -int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionLevel) +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LZ4F_compressionContext_t ctx; +} cRess_t; + +static cRess_t LZ4IO_createCResources(void) +{ + const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + cRess_t ress; + LZ4F_errorCode_t errorCode; + + errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); + + /* Allocate Memory */ + ress.srcBuffer = malloc(blockSize); + ress.srcBufferSize = blockSize; + ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL); /* cover worst case */ + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory"); + + return ress; +} + +static void LZ4IO_freeCResources(cRess_t ress) +{ + LZ4F_errorCode_t errorCode; + free(ress.srcBuffer); + free(ress.dstBuffer); + errorCode = LZ4F_freeCompressionContext(ress.ctx); + if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); +} + +/* + * LZ4IO_compressFilename_extRess() + * result : 0 : compression completed correctly + * 1 : missing or pb opening srcFileName + */ +static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName, const char* dstFileName, int compressionLevel) { unsigned long long filesize = 0; unsigned long long compressedfilesize = 0; - char* in_buff; - char* out_buff; - FILE* finput; - FILE* foutput; - clock_t start, end; - int blockSize; - size_t sizeCheck, headerSize, readSize, outBuffSize; - LZ4F_compressionContext_t ctx; - LZ4F_errorCode_t errorCode; + FILE* srcFile; + FILE* dstFile; + void* const srcBuffer = ress.srcBuffer; + void* const dstBuffer = ress.dstBuffer; + const size_t dstBufferSize = ress.dstBufferSize; + const size_t blockSize = (size_t)LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + size_t sizeCheck, headerSize, readSize; + LZ4F_compressionContext_t ctx = ress.ctx; /* just a pointer */ LZ4F_preferences_t prefs; /* Init */ - start = clock(); memset(&prefs, 0, sizeof(prefs)); - if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3; - errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); - get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4IO_GetBlockSize_FromBlockId (g_blockSizeId); + + /* File check */ + if (LZ4IO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1; /* Set compression parameters */ prefs.autoFlush = 1; prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = (blockMode_t)g_blockIndependence; - prefs.frameInfo.blockSizeID = (blockSizeID_t)g_blockSizeId; - prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_streamChecksum; + prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence; + prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId; + prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum; if (g_contentSizeFlag) { - unsigned long long fileSize = LZ4IO_GetFileSize(input_filename); + unsigned long long fileSize = LZ4IO_GetFileSize(srcFileName); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ + if (fileSize==0) + DISPLAYLEVEL(3, "Warning : cannot determine uncompressed frame content size \n"); } - /* Allocate Memory */ - in_buff = (char*)malloc(blockSize); - outBuffSize = LZ4F_compressBound(blockSize, &prefs); - out_buff = (char*)malloc(outBuffSize); - if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); - - /* Write Archive Header */ - headerSize = LZ4F_compressBegin(ctx, out_buff, outBuffSize, &prefs); - if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); - sizeCheck = fwrite(out_buff, 1, headerSize, foutput); - if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); - compressedfilesize += headerSize; - /* read first block */ - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + readSize = fread(srcBuffer, (size_t)1, blockSize, srcFile); filesize += readSize; - /* Main Loop */ - while (readSize>0) + /* single-block file */ + if (readSize < blockSize) { - size_t outSize; - - /* Compress Block */ - outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL); - if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); - compressedfilesize += outSize; - DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + /* Compress in single pass */ + size_t cSize = LZ4F_compressFrame(dstBuffer, dstBufferSize, srcBuffer, readSize, &prefs); + if (LZ4F_isError(cSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(cSize)); + compressedfilesize += cSize; + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100); /* avoid division by zero */ /* Write Block */ - sizeCheck = fwrite(out_buff, 1, outSize, foutput); - if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); - - /* Read next block */ - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); - filesize += readSize; + sizeCheck = fwrite(dstBuffer, 1, cSize, dstFile); + if (sizeCheck!=cSize) EXM_THROW(35, "Write error : cannot write compressed block"); } - /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, out_buff, outBuffSize, NULL); - if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + else - sizeCheck = fwrite(out_buff, 1, headerSize, foutput); - if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += headerSize; + /* multiple-blocks file */ + { + /* Write Archive Header */ + headerSize = LZ4F_compressBegin(ctx, dstBuffer, dstBufferSize, &prefs); + if (LZ4F_isError(headerSize)) EXM_THROW(32, "File header generation failed : %s", LZ4F_getErrorName(headerSize)); + sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(33, "Write error : cannot write header"); + compressedfilesize += headerSize; + + /* Main Loop */ + while (readSize>0) + { + size_t outSize; - /* Close & Free */ - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - errorCode = LZ4F_freeCompressionContext(ctx); - if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + /* Compress Block */ + outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL); + if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); + compressedfilesize += outSize; + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100); + + /* Write Block */ + sizeCheck = fwrite(dstBuffer, 1, outSize, dstFile); + if (sizeCheck!=outSize) EXM_THROW(35, "Write error : cannot write compressed block"); + + /* Read next block */ + readSize = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile); + filesize += readSize; + } + + /* End of Stream mark */ + headerSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL); + if (LZ4F_isError(headerSize)) EXM_THROW(36, "End of file generation failed : %s", LZ4F_getErrorName(headerSize)); + + sizeCheck = fwrite(dstBuffer, 1, headerSize, dstFile); + if (sizeCheck!=headerSize) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += headerSize; + } + + /* Release files */ + fclose (srcFile); + fclose (dstFile); /* Final Status */ - end = clock(); DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + filesize, compressedfilesize, (double)compressedfilesize/(filesize + !filesize)*100); /* avoid division by zero */ + + return 0; +} + + +int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel) +{ + clock_t start, end; + cRess_t ress; + int issueWithSrcFile = 0; + + /* Init */ + start = clock(); + ress = LZ4IO_createCResources(); + + /* Compress File */ + issueWithSrcFile += LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel); + + /* Free resources */ + LZ4IO_freeCResources(ress); + + /* Final Status */ + end = clock(); { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + double seconds = (double)(end - start) / CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); } - return 0; + return issueWithSrcFile; } #define FNSPACE 30 -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel) +int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionLevel) { int i; - char* outFileName = (char*)malloc(FNSPACE); + int missed_files = 0; + char* dstFileName = (char*)malloc(FNSPACE); size_t ofnSize = FNSPACE; const size_t suffixSize = strlen(suffix); + cRess_t ress; + + /* init */ + ress = LZ4IO_createCResources(); + /* loop on each file */ for (i=0; i<ifntSize; i++) { size_t ifnSize = strlen(inFileNamesTable[i]); - if (ofnSize <= ifnSize+suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } - strcpy(outFileName, inFileNamesTable[i]); - strcat(outFileName, suffix); - LZ4IO_compressFilename(inFileNamesTable[i], outFileName, compressionlevel); + if (ofnSize <= ifnSize+suffixSize+1) { free(dstFileName); ofnSize = ifnSize + 20; dstFileName = (char*)malloc(ofnSize); } + strcpy(dstFileName, inFileNamesTable[i]); + strcat(dstFileName, suffix); + + missed_files += LZ4IO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel); } - free(outFileName); - return 0; + + /* Close & Free */ + LZ4IO_freeCResources(ress); + free(dstFileName); + + return missed_files; } @@ -528,15 +624,105 @@ static unsigned LZ4IO_readLE32 (const void* s) unsigned value32 = srcPtr[0]; value32 += (srcPtr[1]<<8); value32 += (srcPtr[2]<<16); - value32 += (srcPtr[3]<<24); + value32 += ((unsigned)srcPtr[3])<<24; return value32; } -static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) +static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + const size_t* ptrT = bufferT; + size_t bufferSizeT = bufferSize / sizeT; + const size_t* const bufferTEnd = bufferT + bufferSizeT; + static const size_t segmentSizeT = (32 KB) / sizeT; + + if (!g_sparseFileSupport) /* normal write */ + { + size_t sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) EXM_THROW(68, "Write error : cannot write decoded block"); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) + { + int seekResult = fseek(file, 1 GB, SEEK_CUR); + if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) + { + size_t seg0SizeT = segmentSizeT; + size_t nb0T; + int seekResult; + + /* count leading zeros */ + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeT); + + if (nb0T != seg0SizeT) /* not all 0s */ + { + size_t sizeCheck; + seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); + storedSkips = 0; + seg0SizeT -= nb0T; + ptrT += nb0T; + sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file); + if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block"); + } + ptrT += seg0SizeT; + } + + if (bufferSize & maskT) /* size not multiple of sizeT : implies end of block */ + { + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + size_t restSize = bufferSize & maskT; + const char* const restEnd = restStart + restSize; + for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) + { + size_t sizeCheck; + int seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult) EXM_THROW(68, "Skip error (end of block)"); + storedSkips = 0; + sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); + if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block"); + } + } + + return storedSkips; +} + +static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) +{ + char lastZeroByte[1] = { 0 }; + + if (storedSkips>0) /* implies g_sparseFileSupport */ + { + int seekResult; + size_t sizeCheck; + storedSkips --; + seekResult = fseek(file, storedSkips, SEEK_CUR); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + sizeCheck = fwrite(lastZeroByte, 1, 1, file); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); + } +} + + +static unsigned g_magicRead = 0; +static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput) { unsigned long long filesize = 0; char* in_buff; char* out_buff; + unsigned storedSkips = 0; /* Allocate Memory */ in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); @@ -556,7 +742,7 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) blockSize = LZ4IO_readLE32(in_buff); /* Convert to Little Endian */ if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) { /* Cannot read next block : maybe new stream ? */ - fseek(finput, -4, SEEK_CUR); + g_magicRead = blockSize; break; } @@ -570,10 +756,11 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) filesize += decodeSize; /* Write Block */ - sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); - if (sizeCheck != (size_t)decodeSize) EXM_THROW(54, "Write error : cannot write decoded block into output\n"); + storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, decodeSize, storedSkips); } + LZ4IO_fwriteSparseEnd(foutput, storedSkips); + /* Free */ free(in_buff); free(out_buff); @@ -582,172 +769,129 @@ static unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) } -static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) + +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + LZ4F_decompressionContext_t dCtx; +} dRess_t; + +static const size_t LZ4IO_dBufferSize = 64 KB; + +static dRess_t LZ4IO_createDResources(void) { - unsigned long long filesize = 0; - void* inBuff; - void* outBuff; -# define HEADERMAX 20 - char headerBuff[HEADERMAX]; - size_t sizeCheck; - const size_t inBuffSize = 256 KB; - const size_t outBuffSize = 256 KB; - LZ4F_decompressionContext_t ctx; + dRess_t ress; LZ4F_errorCode_t errorCode; - unsigned storedSkips = 0; /* init */ - errorCode = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode)); - LZ4IO_writeLE32(headerBuff, LZ4IO_MAGICNUMBER); /* regenerated here, as it was already read from finput */ + errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); /* Allocate Memory */ - inBuff = malloc(256 KB); - outBuff = malloc(256 KB); - if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory"); + ress.srcBufferSize = LZ4IO_dBufferSize; + ress.srcBuffer = malloc(ress.srcBufferSize); + ress.dstBufferSize = LZ4IO_dBufferSize; + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory"); + + return ress; +} + +static void LZ4IO_freeDResources(dRess_t ress) +{ + LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx); + if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + free(ress.srcBuffer); + free(ress.dstBuffer); +} - /* Init feed with magic number (already consumed from FILE) */ + +static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile) +{ + unsigned long long filesize = 0; + LZ4F_errorCode_t nextToLoad; + unsigned storedSkips = 0; + + /* Init feed with magic number (already consumed from FILE* sFile) */ { - size_t inSize = 4; - size_t outSize=0; - LZ4IO_writeLE32(inBuff, LZ4IO_MAGICNUMBER); - errorCode = LZ4F_decompress(ctx, outBuff, &outSize, inBuff, &inSize, NULL); - if (LZ4F_isError(errorCode)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(errorCode)); + size_t inSize = MAGICNUMBER_SIZE; + size_t outSize= 0; + LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER); + nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad)); } - /* Main Loop */ - for (;;) + for (;nextToLoad;) { size_t readSize; size_t pos = 0; + size_t decodedBytes = ress.dstBufferSize; /* Read input */ - readSize = fread(inBuff, 1, inBuffSize, finput); - if (!readSize) break; /* empty file or stream */ + if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize; + readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile); + if (!readSize) + break; /* empty file or stream */ - while (pos < readSize) + while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) /* still to read, or still to flush */ { /* Decode Input (at least partially) */ size_t remaining = readSize - pos; - size_t decodedBytes = outBuffSize; - errorCode = LZ4F_decompress(ctx, outBuff, &decodedBytes, (char*)inBuff+pos, &remaining, NULL); - if (LZ4F_isError(errorCode)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(errorCode)); + decodedBytes = ress.dstBufferSize; + nextToLoad = LZ4F_decompress(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad)); pos += remaining; + if (!nextToLoad) break; if (decodedBytes) { /* Write Block */ filesize += decodedBytes; - if (g_sparseFileSupport) - { - size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */ - size_t* oBuffPosT = oBuffStartT; - size_t oBuffSizeT = decodedBytes / sizeT; - size_t* const oBuffEndT = oBuffStartT + oBuffSizeT; - static const size_t bs0T = (32 KB) / sizeT; - while (oBuffPosT < oBuffEndT) - { - size_t seg0SizeT = bs0T; - size_t nb0T; - int seekResult; - if (seg0SizeT > oBuffSizeT) seg0SizeT = oBuffSizeT; - oBuffSizeT -= seg0SizeT; - for (nb0T=0; (nb0T < seg0SizeT) && (oBuffPosT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeT); - if (storedSkips > 1 GB) /* avoid int overflow */ - { - seekResult = fseek(foutput, 1 GB, SEEK_CUR); - if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)"); - storedSkips -= 1 GB; - } - if (nb0T != seg0SizeT) /* not all 0s */ - { - seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(68, "Skip error (sparse file)"); - storedSkips = 0; - seg0SizeT -= nb0T; - oBuffPosT += nb0T; - sizeCheck = fwrite(oBuffPosT, sizeT, seg0SizeT, foutput); - if (sizeCheck != seg0SizeT) EXM_THROW(68, "Write error : cannot write decoded block"); - } - oBuffPosT += seg0SizeT; - } - if (decodedBytes & maskT) /* size not multiple of sizeT (necessarily end of block) */ - { - const char* const restStart = (char*)oBuffEndT; - const char* restPtr = restStart; - size_t restSize = decodedBytes & maskT; - const char* const restEnd = restStart + restSize; - for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) - { - int seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult) EXM_THROW(68, "Skip error (end of block)"); - storedSkips = 0; - sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, foutput); - if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(68, "Write error : cannot write decoded end of block"); - } - } - } - else - { - sizeCheck = fwrite(outBuff, 1, decodedBytes, foutput); - if (sizeCheck != decodedBytes) EXM_THROW(68, "Write error : cannot write decoded block"); - } + DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); + storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips); } } - } - if ((g_sparseFileSupport) && (storedSkips>0)) - { - int seekResult; - storedSkips --; - seekResult = fseek(foutput, storedSkips, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); - memset(outBuff, 0, 1); - sizeCheck = fwrite(outBuff, 1, 1, foutput); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); - } + LZ4IO_fwriteSparseEnd(dstFile, storedSkips); - /* Free */ - free(inBuff); - free(outBuff); - errorCode = LZ4F_freeDecompressionContext(ctx); - if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); + if (nextToLoad!=0) + EXM_THROW(67, "Unfinished stream"); return filesize; } -static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE]) +static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE]) { void* buffer = malloc(64 KB); size_t read = 1, sizeCheck; unsigned long long total = MAGICNUMBER_SIZE; + unsigned storedSkips = 0; - sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start"); + sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error"); while (read) { read = fread(buffer, 1, 64 KB, finput); total += read; - sizeCheck = fwrite(buffer, 1, read, foutput); - if (sizeCheck != read) EXM_THROW(50, "Pass-through error"); + storedSkips = LZ4IO_fwriteSparse(foutput, buffer, read, storedSkips); } + LZ4IO_fwriteSparseEnd(foutput, storedSkips); free(buffer); return total; } #define ENDOFSTREAM ((unsigned long long)-1) -static unsigned long long selectDecoder( FILE* finput, FILE* foutput) +static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput) { - unsigned char U32store[MAGICNUMBER_SIZE]; + unsigned char MNstore[MAGICNUMBER_SIZE]; unsigned magicNumber, size; int errorNb; size_t nbReadBytes; @@ -757,33 +901,41 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) nbCalls++; /* Check Archive Header */ - nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ - if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); - magicNumber = LZ4IO_readLE32(U32store); /* Little Endian format */ + if (g_magicRead) + { + magicNumber = g_magicRead; + g_magicRead = 0; + } + else + { + nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes==0) return ENDOFSTREAM; /* EOF */ + if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable"); + magicNumber = LZ4IO_readLE32(MNstore); /* Little Endian format */ + } if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0; /* fold skippable magic numbers */ switch(magicNumber) { case LZ4IO_MAGICNUMBER: - return DEFAULT_DECOMPRESSOR(finput, foutput); + return LZ4IO_decompressLZ4F(ress, finput, foutput); case LEGACY_MAGICNUMBER: DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return decodeLegacyStream(finput, foutput); + return LZ4IO_decodeLegacyStream(finput, foutput); case LZ4IO_SKIPPABLE0: DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - nbReadBytes = fread(U32store, 1, 4, finput); + nbReadBytes = fread(MNstore, 1, 4, finput); if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); - size = LZ4IO_readLE32(U32store); /* Little Endian format */ + size = LZ4IO_readLE32(MNstore); /* Little Endian format */ errorNb = fseek(finput, size, SEEK_CUR); if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); - return selectDecoder(finput, foutput); + return selectDecoder(ress, finput, foutput); EXTENDED_FORMAT; default: if (nbCalls == 1) /* just started */ { if (g_overwrite) - return LZ4IO_passThrough(finput, foutput, U32store); + return LZ4IO_passThrough(finput, foutput, MNstore); EXM_THROW(44,"Unrecognized header : file cannot be decoded"); /* Wrong magic number at the beginning of 1st stream */ } DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); @@ -792,43 +944,95 @@ static unsigned long long selectDecoder( FILE* finput, FILE* foutput) } -int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename) { unsigned long long filesize = 0, decodedSize=0; FILE* finput; FILE* foutput; - clock_t start, end; /* Init */ - start = clock(); - get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput)) + return 1; /* sparse file */ - if (g_sparseFileSupport && foutput) { SET_SPARSE_FILE_MODE(foutput); } + if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); } /* Loop over multiple streams */ do { - decodedSize = selectDecoder(finput, foutput); + decodedSize = selectDecoder(ress, finput, foutput); if (decodedSize != ENDOFSTREAM) filesize += decodedSize; } while (decodedSize != ENDOFSTREAM); /* Final Status */ - end = clock(); DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } /* Close */ fclose(finput); fclose(foutput); - /* Error status = OK */ return 0; } + +int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename) +{ + dRess_t ress; + clock_t start, end; + int missingFiles = 0; + + start = clock(); + + ress = LZ4IO_createDResources(); + missingFiles += LZ4IO_decompressFile_extRess(ress, input_filename, output_filename); + LZ4IO_freeDResources(ress); + + end = clock(); + if (end==start) end=start+1; + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f sec \n", seconds); + } + + return missingFiles; +} + + +int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix) +{ + int i; + int skippedFiles = 0; + int missingFiles = 0; + char* outFileName = (char*)malloc(FNSPACE); + size_t ofnSize = FNSPACE; + const size_t suffixSize = strlen(suffix); + char* ifnSuffix = (char*)malloc(suffixSize + 1); + dRess_t ress; + + ress = LZ4IO_createDResources(); + + for (i=0; i<ifntSize; i++) + { + size_t ifnSize = strlen(inFileNamesTable[i]); + strcpy(ifnSuffix, inFileNamesTable[i] + ifnSize - suffixSize); + if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); } + if (ifnSize <= suffixSize || strcmp(ifnSuffix, suffix) != 0) + { + DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]); + skippedFiles++; + continue; + } + memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize); + outFileName[ifnSize-suffixSize] = '\0'; + + missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName); + } + + LZ4IO_freeDResources(ress); + free(outFileName); + free(ifnSuffix); + return missingFiles + skippedFiles; +} diff --git a/programs/lz4io.h b/programs/lz4io.h index 75a36e1..11c8fdb 100644 --- a/programs/lz4io.h +++ b/programs/lz4io.h @@ -51,8 +51,9 @@ static char const nulmark[] = "/dev/null"; int LZ4IO_compressFilename (const char* input_filename, const char* output_filename, int compressionlevel); int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename); -int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); +int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel); +int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix); /* ************************************************** */ /* ****************** Parameters ******************** */ diff --git a/visual/2012/datagen/datagen.vcxproj b/visual/2012/datagen/datagen.vcxproj new file mode 100644 index 0000000..ea61533 --- /dev/null +++ b/visual/2012/datagen/datagen.vcxproj @@ -0,0 +1,159 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{D745AE2F-596A-403A-9B91-81A8C6779243}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>datagen</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\programs\datagen.c" />
+ <ClCompile Include="..\..\..\programs\datagencli.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\programs\datagen.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/datagen/datagen.vcxproj.filters b/visual/2012/datagen/datagen.vcxproj.filters new file mode 100644 index 0000000..822d9bc --- /dev/null +++ b/visual/2012/datagen/datagen.vcxproj.filters @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Fichiers sources">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Fichiers d%27en-tête">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Fichiers de ressources">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\programs\datagen.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\programs\datagencli.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\programs\datagen.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ </ItemGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/frametest/frametest.vcxproj b/visual/2012/frametest/frametest.vcxproj new file mode 100644 index 0000000..b86f7de --- /dev/null +++ b/visual/2012/frametest/frametest.vcxproj @@ -0,0 +1,166 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>frametest</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\programs\frametest.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/frametest/frametest.vcxproj.filters b/visual/2012/frametest/frametest.vcxproj.filters new file mode 100644 index 0000000..19c3d15 --- /dev/null +++ b/visual/2012/frametest/frametest.vcxproj.filters @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Fichiers sources">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Fichiers d%27en-tête">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Fichiers de ressources">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4frame.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4hc.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\xxhash.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\programs\frametest.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4hc.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\xxhash.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ </ItemGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/fullbench/fullbench.vcxproj b/visual/2012/fullbench/fullbench.vcxproj new file mode 100644 index 0000000..11f8f45 --- /dev/null +++ b/visual/2012/fullbench/fullbench.vcxproj @@ -0,0 +1,166 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>fullbench</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\programs\fullbench.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/fullbench/fullbench.vcxproj.filters b/visual/2012/fullbench/fullbench.vcxproj.filters new file mode 100644 index 0000000..cdcf599 --- /dev/null +++ b/visual/2012/fullbench/fullbench.vcxproj.filters @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Fichiers sources">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Fichiers d%27en-tête">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Fichiers de ressources">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4frame.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4hc.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\xxhash.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\programs\fullbench.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4hc.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\xxhash.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ </ItemGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2012/fuzzer/fuzzer.vcxproj index 9ba4fab..4da822f 100644 --- a/visual/2012/fuzzer/fuzzer.vcxproj +++ b/visual/2012/fuzzer/fuzzer.vcxproj @@ -89,6 +89,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -102,6 +103,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -117,6 +119,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -134,6 +137,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.user b/visual/2012/fuzzer/fuzzer.vcxproj.user deleted file mode 100644 index 7cbb321..0000000 --- a/visual/2012/fuzzer/fuzzer.vcxproj.user +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <PropertyGroup />
-</Project>
\ No newline at end of file diff --git a/visual/2012/lz4.sln b/visual/2012/lz4.sln index 2d07075..80d1a54 100644 --- a/visual/2012/lz4.sln +++ b/visual/2012/lz4.sln @@ -5,40 +5,60 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{ EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
- Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
- Release|Mixed Platforms = Release|Mixed Platforms
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.Build.0 = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.ActiveCfg = Release|Win32
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.Build.0 = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.Build.0 = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.ActiveCfg = Release|Win32
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.Build.0 = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
diff --git a/visual/2012/lz4.v11.suo b/visual/2012/lz4.v11.suo Binary files differdeleted file mode 100644 index 8321984..0000000 --- a/visual/2012/lz4.v11.suo +++ /dev/null diff --git a/visual/2012/lz4/lz4.vcxproj b/visual/2012/lz4/lz4.vcxproj index 5540d5d..8ac9387 100644 --- a/visual/2012/lz4/lz4.vcxproj +++ b/visual/2012/lz4/lz4.vcxproj @@ -89,6 +89,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -102,6 +103,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -117,6 +119,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -134,6 +137,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
diff --git a/visual/2012/lz4/lz4.vcxproj.user b/visual/2012/lz4/lz4.vcxproj.user deleted file mode 100644 index 7cbb321..0000000 --- a/visual/2012/lz4/lz4.vcxproj.user +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <PropertyGroup />
-</Project>
\ No newline at end of file |