diff options
| author | Yann Collet <yann.collet.73@gmail.com> | 2015-04-03 22:49:20 (GMT) |
|---|---|---|
| committer | Yann Collet <yann.collet.73@gmail.com> | 2015-04-03 22:49:20 (GMT) |
| commit | 157a739f7ade8b3f41cb2f6aeaa9ca30c376c7ca (patch) | |
| tree | 9dc9ff295366749d4af27cf70e3c8acbe4c990ff | |
| parent | 8437a0e43aa5086ec2c23591909e118a85c3d8dd (diff) | |
| parent | 43eaf8f33361555f5cbe4306c17a7ca7869c61c8 (diff) | |
| download | lz4-157a739f7ade8b3f41cb2f6aeaa9ca30c376c7ca.zip lz4-157a739f7ade8b3f41cb2f6aeaa9ca30c376c7ca.tar.gz lz4-157a739f7ade8b3f41cb2f6aeaa9ca30c376c7ca.tar.bz2 | |
Merge pull request #74 from Cyan4973/fastMode
Fast mode
33 files changed, 1341 insertions, 277 deletions
diff --git a/.travis.yml b/.travis.yml index 8fabb1c..8f45f71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ before_install: - sudo apt-get install -qq valgrind env: - - LZ4_TRAVIS_CI_ENV=dist - LZ4_TRAVIS_CI_ENV=travis-install - LZ4_TRAVIS_CI_ENV=streaming-examples - LZ4_TRAVIS_CI_ENV=cmake diff --git a/LZ4_Frame_Format.html b/LZ4_Frame_Format.html deleted file mode 100644 index 6622dbb..0000000 --- a/LZ4_Frame_Format.html +++ /dev/null @@ -1 +0,0 @@ -<html><head><title>LZ4 Framing format - v1.5.0</title><meta content="text/html; charset=UTF-8" http-equiv="content-type"><style type="text/css">@import url('https://themes.googleusercontent.com/fonts/css?kit=wAPX1HepqA24RkYW1AuHYA');ol{margin:0;padding:0}.c2{font-size:10pt;font-family:"Courier New";font-weight:bold}.c10{max-width:453.6pt;background-color:#ffffff;padding:70.8pt 70.8pt 70.8pt 70.8pt}.c4{line-height:1.0;height:11pt;padding-bottom:0pt}.c0{direction:ltr;margin-left:18pt}.c13{line-height:1.0;padding-bottom:0pt}.c8{color:inherit;text-decoration:inherit}.c3{color:#1155cc;text-decoration:underline}.c6{text-decoration:underline;font-weight:bold}.c14{font-weight:bold}.c18{font-family:"Courier New"}.c16{font-size:18pt}.c12{margin-left:36pt}.c11{font-size:14pt}.c5{height:11pt}.c9{text-align:center}.c7{text-decoration:underline}.c17{color:#0000ff}.c1{direction:ltr}.c15{font-style:italic}.title{padding-top:12pt;line-height:1.15;text-align:center;color:#000000;font-size:16pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}.subtitle{padding-top:0pt;line-height:1.15;text-align:center;color:#000000;font-size:11pt;font-family:"Arial";padding-bottom:3pt}li{color:#000000;font-size:11pt;font-family:"Calibri"}p{color:#000000;font-size:11pt;margin:0;font-family:"Calibri"}h1{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:16pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}h2{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-style:italic;font-size:14pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}h3{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:13pt;font-family:"Arial";font-weight:bold;padding-bottom:3pt}h4{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:14pt;font-family:"Calibri";font-weight:bold;padding-bottom:3pt}h5{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-style:italic;font-size:13pt;font-family:"Calibri";font-weight:bold;padding-bottom:3pt}h6{padding-top:12pt;line-height:1.15;text-align:left;color:#000000;font-size:11pt;font-family:"Calibri";font-weight:bold;padding-bottom:3pt}</style></head><body class="c10"><hr><p class="c1 c9"><span class="c14 c16">LZ4 </span><span class="c14 c16">Framing </span><span class="c14 c16">Format</span></p><hr><p class="c5 c1"><span class="c14 c16"></span></p><p class="c1"><span class="c6 c11">Notices</span></p><p class="c1"><span>Copyright (c) 2013-2015 Yann Collet</span></p><p class="c1"><span>Permission is granted to copy and distribute this document for any purpose and without charge, including translations into other languages and incorporation into compilations, provided that the copyright notice and this notice are preserved, and that any substantive changes or deletions from the original are clearly marked.</span></p><p class="c1"><span class="c6 c11">Version</span></p><p class="c1"><span>1.5.0</span></p><h1 class="c1"><a name="h.2z5bl598dfq9"></a><span>Introduction</span></h1><p class="c1"><span>The purpose of this document is to define a lossless compressed data format, that is independent of CPU type, operating system, file system and character set, suitable for File compression, Pipe and streaming compression using the LZ4 algorithm : </span><span class="c7 c17"><a class="c8" href="">http://code.google.com/p/lz4/</a></span></p><p class="c1"><span>The data can be produced or consumed, even for an arbitrarily long sequentially presented input data stream, using only an a priori bounded amount of intermediate storage, and hence can be used in data communications. The format uses the LZ4 compression method, and optional </span><span class="c3"><a class="c8" href="http://code.google.com/p/xxhash/">xxHash-32</a></span><span> checksum method, for detection of data corruption.</span></p><p class="c1"><span>The data format defined by this specification does not attempt to allow random access to compressed data.</span></p><p class="c1"><span>This specification is intended for use by implementers of software to compress data into LZ4 format and/or decompress data from LZ4 format. The text of the specification assumes a basic background in programming at the level of bits and other primitive data representations.</span></p><p class="c1"><span>Unless otherwise indicated below, </span><span>a compliant compressor must produce data sets that conform to the specifications presented here. It doesn’t need to support all options though.</span></p><p class="c1"><span>A</span><span> compliant decompressor must be able to decompress </span><span>at least one working </span><span>set of parameters that conforms to the specifications presented here</span><span>. It may also ignore checksums. Whenever it does not support a specific parameter used within the compressed stream, it must produce a non-ambiguous error code and associated error message explaining which parameter is unsupported.</span></p><p class="c1"><span>Distribution of this document is unlimited.</span></p><p class="c4 c1"><span></span></p><hr style="page-break-before:always;display:none;"><p class="c1 c4"><span></span></p><p class="c1 c13"><span class="c6 c11">Summary </span><span class="c6">:</span></p><p class="c4 c1"><span></span></p><p class="c0"><span class="c3"><a class="c8" href="#h.2z5bl598dfq9">Introduction</a></span></p><p class="c0"><span class="c3">General structure of </span><span class="c3"><a class="c8" href="#h.1615sutikt7e">LZ4 Framing Format</a></span></p><p class="c12 c1"><span class="c3">Frame </span><span class="c3"><a class="c8" href="#h.uof0plru1f66">Descriptor</a></span></p><p class="c1 c12"><span class="c3"><a class="c8" href="#h.u8dkhfnwqyg">Data Blocks</a></span></p><p class="c0"><span class="c3"><a class="c8" href="#h.152pfqac8luc">Skippable </a></span><span class="c3">Frames</span></p><p class="c0"><span class="c3"><a class="c8" href="#h.ujcdmapf87vn">Legacy format</a></span></p><p class="c0"><span class="c3"><a class="c8" href="#h.zij6fhosmkvv">Appendix</a></span></p><p class="c5 c1"><span></span></p><p class="c5 c1"><span class="c6 c11"></span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.1615sutikt7e"></a><span class="c7">General Structure of </span><span class="c7">LZ4 Framing format</span></h1><p class="c5 c9 c1"><span class="c6 c11"></span></p><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 408.00px; height: 106.00px;"><img alt="" src="images/image05.png" style="width: 408.00px; height: 106.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c5 c1"><span class="c6 c11"></span></p><p class="c1"><span class="c6">Magic Number</span></p><p class="c1"><span>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br>Value : </span><span class="c14 c18">0x184D2204</span></p><p class="c5 c1"><span class="c2"></span></p><p class="c1"><span class="c6">Frame D</span><span class="c6">escriptor</span></p><p class="c1"><span>3</span><span> to 1</span><span>1</span><span> Bytes, to be detailed </span><span>in the next part.</span><span><br>Most important part of the spec.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Data Blocks</span></p><p class="c1"><span>To be detailed later on.<br>That’s where compressed data is stored.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">EndMark</span></p><p class="c1"><span>The flow of </span><span>blocks </span><span>ends when the last data block has a size of “</span><span class="c14">0</span><span>”. </span><span><br></span><span>The size is expressed as </span><span>a </span><span>32-bits value.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Content Checksum</span></p><p class="c1"><span>Content Checksum verify that the full content has been decoded correctly.<br>The content checksum is the result of </span><span class="c3"><a class="c8" href="http://code.google.com/p/xxhash/">xxh32()</a></span><span> hash function digesting the original (decoded) data as input, and a seed of zero.<br>Content checksum is only present when its </span><span class="c3"><a class="c8" href="#id.s5zerkv6retr">associated flag </a></span><span>is set in the framing descriptor. Content Checksum validates the result, that all blocks were fully transmitted in the correct order and without error, and also that the encoding/decoding process itself generated no distortion. Its usage is recommended. </span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Frame Concatenation</span></p><p class="c1"><span>In some circumstances, it may be preferable </span><span>to append multiple frames, </span><span>for example </span><span>in order to add new data to an existing compressed file without re-framing it.</span></p><p class="c1"><span>In such case, each frame has its own set of descriptor flags. Each frame is considered independent. The only relation between frames is their sequential order.</span></p><p class="c1"><span>The ability to decode multiple concatenated frames within a single stream or file is left outside of this specification. As an example, the reference lz4 command line utility behavior is to decode all concatenated frames in their sequential order. </span></p><p class="c5 c1"><span></span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span></span></p><h2 class="c1"><a name="h.uof0plru1f66"></a><span class="c7">Frame </span><span class="c7">Descriptor</span></h2><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 226.00px; height: 106.00px;"><img alt="" src="images/image00.png" style="width: 226.00px; height: 106.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 606.00px; height: 125.33px;"><img alt="" src="images/image01.png" style="width: 606.00px; height: 125.33px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>The descriptor uses a minimum of </span><span>3</span><span> bytes</span><span>, and up to 11 bytes depending on optional parameters.</span><span><br>In the picture, bit 7 is highest bit, while bit 0 is lowest.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Version Number :</span></p><p class="c1"><span>2-bits field, </span><span class="c6">must</span><span class="c14"> </span><span>be set to “</span><span class="c14">01</span><span>”.<br>Any other value cannot be decoded by this </span><span>version of the specification.</span><span><br>Other version numbers will use different flag layouts.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Block </span><span class="c6">Independence </span><span class="c6">flag :</span></p><p class="c1"><span>If this flag is set to “1”</span><span>, blocks are independent, and can therefore be decoded independently, in parallel.<br>If this flag is set to “</span><span>0</span><span>”, each block depends on previous ones for decoding (up to LZ4 window size, which is 64 KB). In this case, it’s necessary to decode all blocks in sequence.</span></p><p class="c1"><span>Block </span><span>dependency</span><span> improves compression ratio, especially for small blocks. On the other hand, it makes jumps or multi-threaded decoding impossible.</span></p><p class="c5 c1"><span></span></p><a href="#" name="id.r4mqxzdxswxz"></a><p class="c1"><span class="c6">Block checksum flag :</span></p><p class="c1"><span>If this flag is set, e</span><span>ach data block will be followed by a 4-bytes checksum, calculated by using the xxHash-32 algorithm on the raw (compressed) data block.<br>The intention is to detect data corruption (storage or transmission errors) immediately, before decoding.<br>Block ch</span><span>ecksum usage is optional.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Content </span><span class="c6">Size flag :</span></p><p class="c1"><span>If this flag is set, the original (uncompressed) size of data included within </span><span>the frame</span><span> will be present as an 8 bytes unsigned value, litt</span><span>le endian format, </span><span>after the flags.</span></p><p class="c1"><span>Recommended </span><span>value : “</span><span class="c14">0</span><span>” (not present)</span></p><p class="c1 c5"><span></span></p><p class="c1"><span class="c6">Content checksum flag :</span></p><p class="c1"><span>If this flag is set, a</span><span class="c3"><a class="c8" href="#id.q3958klk497z"> content checksum</a></span><span> will be appended after the EoS mark.</span></p><p class="c1"><span>Recommended value : “</span><span class="c14">1</span><span>” (content checksum is present)</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Block Maximum Size :</span></p><p class="c1"><span>This information is intended to help the decoder allocate the right amount of memory.<br>Size here refers to the original (uncompressed) data size.<br>Block Maximum Size </span><span>is</span><span> one value among the fol</span><span>lowing table : </span></p><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 331.00px; height: 56.00px;"><img alt="" src="images/image03.png" style="width: 331.00px; height: 56.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>The decoder may refuse to allocate block sizes above a (system-specific) size.<br>Unused values may be used in a future revision of the spec.<br>A decoder conformant to the current version of the spec is only able to decode blocksizes defined in this spec.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Reserved bits :</span></p><p class="c1"><span>Value of reserved bits </span><span class="c6">must </span><span>be </span><span class="c14">0</span><span> (zero).<br>Reserved bit might be used in a future version of the specification, to enable any (yet-to-decide) optional feature.<br>If this happens, a decoder respecting the current version of the specification shall not be able to decode such a frame.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Content Size</span></p><p class="c1"><span>This is the original (uncompressed) size. <br>This information is optional, and only present if the </span><span class="c3"><a class="c8" href="#id.tqyy099hxhnn">associated flag is set</a></span><span>.<br>Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes.<br>Format is Little endian.<br>This field has no impact on decoding, it just informs the decoder how much data the frame holds (for example, to display it during decoding process, or for verification purpose). It can be safely skipped by a conformant decoder.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Header Checksum :</span></p><p class="c1"><span>One-byte checksum of all descriptor fields, including optional ones when present.<br>The byte is second byte of </span><span class="c3"><a class="c8" href="http://code.google.com/p/xxhash/">xxh32()</a></span><span> : { (xxh32()>>8) & 0xFF } ,<br>using zero as a seed, <br>and the full Frame Descriptor as an input (</span><span class="c7 c15">including</span><span> optional fields when they are present).<br>A different checksum indicates an error in the descriptor.</span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h2 class="c1"><a name="h.u8dkhfnwqyg"></a><span class="c7">Data Blocks</span></h2><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 317.00px; height: 90.00px;"><img alt="" src="images/image02.png" style="width: 317.00px; height: 90.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c5 c1"><span class="c6 c11"></span></p><p class="c1"><span class="c6">Block</span><span class="c6"> Size</span></p><p class="c1"><span>Th</span><span>is</span><span> field uses </span><span class="c14">4</span><span class="c14">-bytes, </span><span>f</span><span>ormat is </span><span class="c7">little-endian</span><span>.</span></p><p class="c1"><span>The highest bit is “</span><span class="c14">1</span><span>” if data in the block is uncompressed.</span></p><p class="c1"><span>The highest bit is “</span><span class="c14">0</span><span>” if data in the block is compressed by LZ4.</span></p><p class="c1"><span>All other bits give the size, in bytes, of the following data block (the size does not include the checksum if present).</span></p><p class="c1"><span>Block Size shall never be larger than Block Maximum Size. Such a thing could happen when the original data is incompressible. In this case, such a data block shall be passed in uncompressed format.</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Data</span></p><p class="c1"><span>Where the actual data to decode stands. It might be compressed or not, depending on previous field indications.<br>Uncompressed size of Data can be any size, up to “block maximum size”. <br>Note that data block is not necessarily </span><span>full </span><span>: an arbitrary “flush” may happen anytime. Any block can be </span><span>“partially filled”.</span></p><p class="c5 c1"><span></span></p><a href="#" name="id.3p4pcqe6ab8n"></a><p class="c1"><span class="c6">Block checksum :</span></p><p class="c1"><span>Only present if the </span><span class="c3"><a class="c8" href="#id.r4mqxzdxswxz">associated flag is set</a></span><span>.<br>This is a 4-bytes checksum value, in little endian format, <br>calculated by using the xxHash-32 algorithm </span><span class="c7">on the raw (undecoded) data block</span><span>, <br>and a seed of zero.</span><span><br>The intention is to detect data corruption (storage or transmission errors) </span><span class="c15">before </span><span>decoding.</span></p><p class="c1"><span>Block checksum is cumulative with Content checksum.</span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.152pfqac8luc"></a><span class="c7">Skippable Frames</span></h1><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 285.00px; height: 106.00px;"><img alt="LZ4 Framing Format - Skippable Frame.png" src="images/image04.png" style="width: 285.00px; height: 106.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>Skippable frames allow the integration of user-defined data into a flow of concatenated frames.<br>Its design is pretty straightforward, with the sole objective to allow the decoder to quickly skip over user-defined data and continue decoding.</span></p><p class="c1"><span>For the purpose of facilitating identification, it is discouraged to start a flow of concatenated frames with a skippable frame. If there is a need to start such a flow with some user data encapsulated into a skippable frame, it’s recommended to start will a zero-byte LZ4 frame followed by a skippable frame. This will make it easier for file type identifiers.</span></p><p class="c1"><span> </span></p><p class="c1"><span class="c6">Magic Number</span></p><p class="c1"><span>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br>Value : </span><span class="c2">0x184D2A5X</span><span>, which means any value from</span><span class="c2"> 0x184D2A50 to 0x184D2A5F.</span><span> All 16 values are valid to identify a skippable frame.<br></span></p><p class="c1"><span class="c6">Frame Size</span><span class="c6"> </span></p><p class="c1"><span>This is the size, in bytes, of the following User Data (without including the magic number nor the size field itself).<br>4 Bytes, </span><span class="c7">Little endian</span><span> format, unsigned 32-bits.<br>This means User Data can’t be bigger than (2^32-1) Bytes.<br></span></p><p class="c1"><span class="c6">User Data</span></p><p class="c1"><span>User Data can be anything. Data will just be skipped by the decoder. </span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.ujcdmapf87vn"></a><span class="c7">Legacy frame</span></h1><p class="c9 c1"><span style="overflow: hidden; display: inline-block; margin: 0.00px 0.00px; border: 0.00px solid #000000; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px); width: 570.00px; height: 90.00px;"><img alt="" src="images/image06.png" style="width: 570.00px; height: 90.00px; margin-left: 0.00px; margin-top: 0.00px; transform: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></p><p class="c1"><span>The Legacy frame format was defined into the initial versions of “LZ4Demo”.<br>Newer compressors should not use this format anymore, since it is too restrictive.<br>It is recommended that decompressors shall be able to decode this format during the transition period.</span></p><p class="c1"><span>Main properties of legacy format :<br>- Fixed block size : </span><span>8 MB</span><span>.<br>- All blocks must be completely filled, except the last one.<br>- All blocks are always compressed, even when compression is detri</span><span>mental.</span><span><br>- The last block is detected either because it is followed by the “EOF” (End of File) mark</span><span>, or because it is followed by a known Frame Magic Number.</span><span><br>- No checksum<br>- Convention is Little endian</span></p><p class="c5 c1"><span></span></p><p class="c1"><span class="c6">Magic Number</span></p><p class="c1"><span>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br>Value : </span><span class="c2">0x184C2102<br></span></p><p class="c1"><span class="c6">Block Compressed Size</span></p><p class="c1"><span>This is the size, in bytes, of the following compressed data block.<br>4 Bytes, </span><span class="c7">Little endian</span><span> format.<br></span></p><p class="c1"><span class="c6">Data</span></p><p class="c1"><span>Where the actual data stands. <br>Data is </span><span class="c7">always</span><span> compressed, even when compression is detrimental (i.e. larger than original size).</span></p><hr style="page-break-before:always;display:none;"><p class="c5 c1"><span class="c6 c11"></span></p><h1 class="c1"><a name="h.zij6fhosmkvv"></a><span class="c6">Appendix </span><span> </span></h1><p class="c1"><span class="c11">Version changes</span></p><p class="c1"><span>1.4.1 : changed wording from “stream” to “frame”</span></p><p class="c1"><span>1.4 : added skippable streams, re-added stream checksum </span></p><p class="c1"><span>1.3 : modified header checksum</span></p><p class="c1"><span>1.2 : reduced choice of “block size”, to postpone decision on “dynamic size of BlockSize Field”.</span></p><p class="c1"><span>1.1 : optional fields are now part of the descriptor</span></p><p class="c1"><span>1.0 : changed “block size” specification, adding a compressed/uncompressed flag</span></p><p class="c1"><span>0.9 : reduced scale of “block maximum size” table</span></p><p class="c1"><span>0.8 : removed : high compression flag</span></p><p class="c1"><span>0.7 : removed : stream checksum</span></p><p class="c1"><span>0.6 : settled : stream size uses 8 bytes, endian convention is little endian</span></p><p class="c1"><span>0.5: added copyright notice</span></p><p class="c1"><span>0.4 : changed format to Google Doc compatible OpenDocument</span></p></body></html>
\ No newline at end of file @@ -31,7 +31,7 @@ # ################################################################ # Version number -export VERSION=128 +export VERSION=129 export RELEASE=r$(VERSION) DESTDIR?= @@ -41,24 +41,6 @@ LIBDIR ?= $(PREFIX)/lib INCLUDEDIR=$(PREFIX)/include PRGDIR = programs LZ4DIR = lib -DISTRIBNAME=lz4-$(RELEASE).tar.gz - -TEXT = $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4hc.h \ - $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h \ - $(LZ4DIR)/xxhash.c $(LZ4DIR)/xxhash.h \ - $(LZ4DIR)/liblz4.pc.in $(LZ4DIR)/Makefile $(LZ4DIR)/LICENSE \ - Makefile lz4_block_format.txt LZ4_Frame_Format.html NEWS README.md \ - cmake_unofficial/CMakeLists.txt \ - $(PRGDIR)/fullbench.c $(PRGDIR)/lz4cli.c \ - $(PRGDIR)/datagen.c $(PRGDIR)/datagen.h $(PRGDIR)/datagencli.c $(PRGDIR)/fuzzer.c \ - $(PRGDIR)/lz4io.c $(PRGDIR)/lz4io.h \ - $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ - $(PRGDIR)/lz4.1 \ - $(PRGDIR)/Makefile $(PRGDIR)/COPYING -NONTEXT = images/image00.png images/image01.png images/image02.png \ - images/image03.png images/image04.png images/image05.png \ - images/image06.png -SOURCES = $(TEXT) $(NONTEXT) # Select test target for Travis CI's Build Matrix @@ -86,7 +68,6 @@ lz4programs: @cd $(PRGDIR); $(MAKE) -e clean: - @rm -f $(DISTRIBNAME) *.sha1 > $(VOID) @cd $(PRGDIR); $(MAKE) clean > $(VOID) @cd $(LZ4DIR); $(MAKE) clean > $(VOID) @cd examples; $(MAKE) clean > $(VOID) @@ -108,24 +89,6 @@ uninstall: travis-install: sudo $(MAKE) install -dist: clean - @install -dD -m 700 lz4-$(RELEASE)/lib/ - @install -dD -m 700 lz4-$(RELEASE)/programs/ - @install -dD -m 700 lz4-$(RELEASE)/cmake_unofficial/ - @install -dD -m 700 lz4-$(RELEASE)/images/ - @for f in $(TEXT); do \ - tr -d '\r' < $$f > .tmp; \ - install -m 600 .tmp lz4-$(RELEASE)/$$f; \ - done - @rm .tmp - @for f in $(NONTEXT); do \ - install -m 600 $$f lz4-$(RELEASE)/$$f; \ - done - @tar -czf $(DISTRIBNAME) lz4-$(RELEASE)/ - @rm -rf lz4-$(RELEASE) - @sha1sum $(DISTRIBNAME) > $(DISTRIBNAME).sha1 - @echo Distribution $(DISTRIBNAME) built - test: @cd $(PRGDIR); $(MAKE) -e test @@ -134,18 +97,18 @@ test-travis: $(TRAVIS_TARGET) cmake: @cd cmake_unofficial; cmake CMakeLists.txt; $(MAKE) +gpptest: clean + export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"; $(MAKE) -e all + clangtest: clean - export CC=clang; $(MAKE) all + export CFLAGS=-Werror; export CC=clang; $(MAKE) all staticAnalyze: clean export CFLAGS=-g; scan-build -v $(MAKE) all -gpptest: clean - export CC=g++; export CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align"; $(MAKE) -e all - armtest: clean - export CC=arm-linux-gnueabi-gcc; cd lib; $(MAKE) -e all - export CC=arm-linux-gnueabi-gcc; cd programs; $(MAKE) -e bins + export CFLAGS=-Werror; export CC=arm-linux-gnueabi-gcc; cd lib; $(MAKE) -e all + export CFLAGS=-Werror; export CC=arm-linux-gnueabi-gcc; cd programs; $(MAKE) -e bins streaming-examples: cd examples; $(MAKE) -e test @@ -1,19 +1,24 @@ +r129: +Added : LZ4_compress_fast() +Added : AppVeyor CI environment, for Visual tests - Suggested by Takayuki Matsuoka +Updated: Documentation converted to MarkDown + r128: -New : lz4cli sparse file support -New : command -m, to compress multiple files in a single command -Fixed : Restored lz4hc compression ratio (was slightly lower since r124) -New : lz4 cli supports long commands -New : lz4frame & lz4cli frame content size support -New : lz4frame supports skippable frames -Changed:Default "make install" directory is /usr/local -New : lz4 cli supports "pass-through" mode -New : datagen can generate sparse files -New : scan-build tests -New : g++ compatibility tests -New : arm cross-compilation test -Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48) -Added : Visual project directory -Updated:Man page & Specification +New : lz4cli sparse file support (Requested by Neil Wilson, and contributed by Takayuki Matsuoka) +New : command -m, to compress multiple files in a single command (suggested by Kyle J. Harper) +Fixed : Restored lz4hc compression ratio (slightly lower since r124) +New : lz4 cli supports long commands (suggested by Takayuki Matsuoka) +New : lz4frame & lz4cli frame content size support +New : lz4frame supports skippable frames, as requested by Sergey Cherepanov +Changed: Default "make install" directory is /usr/local, as notified by Ron Johnson +New : lz4 cli supports "pass-through" mode, requested by Neil Wilson +New : datagen can generate sparse files +New : scan-build tests, thanks to kind help by Takayuki Matsuoka +New : g++ compatibility tests +New : arm cross-compilation test, thanks to kind help by Takayuki Matsuoka +Fixed : Fuzzer + frametest compatibility with NetBSD (issue #48, reported by Thomas Klausner) +Added : Visual project directory +Updated: Man page & Specification r127: N/A : added a file on SVN @@ -1,60 +1,68 @@ LZ4 - Extremely fast compression ================================ -LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, scalable with multi-cores CPU. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. -A high compression derivative, called LZ4_HC, is also provided. It trades CPU time for compression ratio. +LZ4 is lossless compression algorithm, +providing compression speed at 400 MB/s per core, +scalable with multi-cores CPU. +It also features an extremely fast decoder, +with speed in multiple GB/s per core, +typically reaching RAM speed limits on multi-core systems. + +A high compression derivative, called LZ4_HC, is also provided. +It trades CPU time for compression ratio. |Branch |Status | |------------|---------| |master | [](https://travis-ci.org/Cyan4973/lz4) | |dev | [](https://travis-ci.org/Cyan4973/lz4) | +|visual | [](https://ci.appveyor.com/project/YannCollet/lz4) | + > **Branch Policy:** > - The "master" branch is considered stable, at all times. -> - The "dev" branch is the one where all contributions must be merged before being promoted to master. -> - If you plan to propose a patch, please commit into the "dev" branch. Direct commit to "master" are not permitted. -> - Feature branches can also exist, for dedicated testing of larger modifications before merge into "dev" branch. +> - The "dev" branch is the one where all contributions must be merged + before being promoted to master. +> + If you plan to propose a patch, please commit into the "dev" branch. + Direct commit to "master" are not permitted. +> - Feature branches can also exist, + for dedicated tests of larger modifications before merge into "dev" branch. Benchmarks ------------------------- -The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.2)](http://encode.ru/threads/1371-Filesystem-benchmark?p=33548&viewfull=1#post33548) compiled with GCC v4.6.1 on Linux Ubuntu 64-bits v11.10, -The reference system uses a Core i5-3340M @2.7GHz. -Benchmark evaluates the compression of reference [Silesia Corpus](http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia) in single-thread mode. - -<table> - <tr> - <th>Compressor</th><th>Ratio</th><th>Compression</th><th>Decompression</th> - </tr> - <tr> - <th>LZ4 (r101)</th><th>2.084</th><th>422 MB/s</th><th>1820 MB/s</th> - </tr> - <tr> - <th>LZO 2.06</th><th>2.106</th><th>414 MB/s</th><th>600 MB/s</th> - </tr> - <tr> - <th>QuickLZ 1.5.1b6</th><th>2.237</th><th>373 MB/s</th><th>420 MB/s</th> - </tr> - <tr> - <th>Snappy 1.1.0</th><th>2.091</th><th>323 MB/s</th><th>1070 MB/s</th> - </tr> - <tr> - <th>LZF</th><th>2.077</th><th>270 MB/s</th><th>570 MB/s</th> - </tr> - <tr> - <th>zlib 1.2.8 -1</th><th>2.730</th><th>65 MB/s</th><th>280 MB/s</th> - </tr> - <tr> - <th>LZ4 HC (r101)</th><th>2.720</th><th>25 MB/s</th><th>2080 MB/s</th> - </tr> - <tr> - <th>zlib 1.2.8 -6</th><th>3.099</th><th>21 MB/s</th><th>300 MB/s</th> - </tr> -</table> - -The LZ4 block compression format is detailed within [lz4_block_format.txt](lz4_block_format.txt). - -For streaming unknown amount of data and compress files of any size, a frame format has been published, and can be consulted within the file LZ4_Frame_Format.html . +The benchmark uses the [Open-Source Benchmark program by m^2 (v0.14.3)] +compiled with GCC v4.8.2 on Linux Mint 64-bits v17. +The reference system uses a Core i5-4300U @1.9GHz. +Benchmark evaluates the compression of reference [Silesia Corpus] +in single-thread mode. + +| Compressor | Ratio | Compression | Decompression | +| ---------- | ----- | ----------- | ------------- | +| memcpy | 1.000 | 4200 MB/s | 4200 MB/s | +| RLE64 v3.0 | 1.029 | 2800 MB/s | 2800 MB/s | +| density -c1 | 1.592 | 700 MB/s | 920 MB/s | +|**LZ4 fast (r129)**| 1.607 |**680 MB/s** | **2220 MB/s** | +|**LZ4 (r129)** |**2.101**|**385 MB/s** | **1850 MB/s** | +| density -c2 | 2.083 | 370 MB/s | 505 MB/s | +| LZO 2.06 | 2.108 | 350 MB/s | 510 MB/s | +| QuickLZ 1.5.1.b6 | 2.238 | 320 MB/s | 380 MB/s | +| Snappy 1.1.0 | 2.091 | 250 MB/s | 960 MB/s | +| density -c3 | 2.370 | 190 MB/s | 185 MB/s | +| zlib 1.2.8 -1 | 2.730 | 59 MB/s | 250 MB/s | +|**LZ4 HC (r129)** |**2.720**| 22 MB/s | **1830 MB/s** | +| zlib 1.2.8 -6 | 3.099 | 18 MB/s | 270 MB/s | + +The LZ4 block compression format is detailed within [lz4_Block_format]. + +Block format doesn't deal with header information, +nor how to handle arbitrarily long files or data streams. +This is the purpose of the Frame format. +Interoperable versions of LZ4 should use the same frame format, +defined into [lz4_Frame_format]. +[Open-Source Benchmark program by m^2 (v0.14.3)]: http://encode.ru/threads/1371-Filesystem-benchmark?p=34029&viewfull=1#post34029 +[Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia +[lz4_Block_format]: lz4_Block_format.md +[lz4_Frame_format]: lz4_Frame_format.md diff --git a/images/image00.png b/images/image00.png Binary files differdeleted file mode 100644 index f4645f2..0000000 --- a/images/image00.png +++ /dev/null diff --git a/images/image01.png b/images/image01.png Binary files differdeleted file mode 100644 index 661c465..0000000 --- a/images/image01.png +++ /dev/null diff --git a/images/image02.png b/images/image02.png Binary files differdeleted file mode 100644 index a909353..0000000 --- a/images/image02.png +++ /dev/null diff --git a/images/image03.png b/images/image03.png Binary files differdeleted file mode 100644 index 8e1b40c..0000000 --- a/images/image03.png +++ /dev/null diff --git a/images/image04.png b/images/image04.png Binary files differdeleted file mode 100644 index e82d02b..0000000 --- a/images/image04.png +++ /dev/null diff --git a/images/image05.png b/images/image05.png Binary files differdeleted file mode 100644 index 3a8006a..0000000 --- a/images/image05.png +++ /dev/null diff --git a/images/image06.png b/images/image06.png Binary files differdeleted file mode 100644 index 69b4ee6..0000000 --- a/images/image06.png +++ /dev/null @@ -44,6 +44,12 @@ #define HEAPMODE 0 /* + * ACCELERATION_DEFAULT : + * Select the value of "acceleration" for LZ4_compress_fast() when parameter == 0 + */ +#define ACCELERATION_DEFAULT 17 + +/* * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : * By default, the source code expects the compiler to correctly optimize * 4-bytes and 8-bytes read on architectures able to handle it efficiently. @@ -113,7 +119,7 @@ # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #else # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# ifdef __GNUC__ +# if defined(__GNUC__) || defined(__clang__) # define FORCE_INLINE static inline __attribute__((always_inline)) # else # define FORCE_INLINE static inline @@ -423,13 +429,6 @@ static const U32 LZ4_skipTrigger = 6; /* Increase this value ==> compression ru /************************************** -* Local Utils -**************************************/ -int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } -int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } - - -/************************************** * Local Structures and types **************************************/ typedef struct { @@ -451,6 +450,14 @@ typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { full = 0, partial = 1 } earlyEnd_directive; +/************************************** +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + + /******************************** * Compression functions @@ -464,7 +471,22 @@ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); } +static const U64 prime5bytes = 889523592379ULL; +static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + const U32 hashMask = (1<<hashLog) - 1; + return ((sequence * prime5bytes) >> (40 - hashLog)) & hashMask; +} + +static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType) +{ + if (LZ4_64bits()) + return LZ4_hashSequence64(sequence, tableType); + return LZ4_hashSequence((U32)sequence, tableType); +} + +static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { @@ -496,15 +518,16 @@ static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t t } static int LZ4_compress_generic( - void* ctx, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, - limitedOutput_directive outputLimited, - tableType_t const tableType, - dict_directive dict, - dictIssue_directive dictIssue) + void* const ctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) { LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; @@ -527,7 +550,7 @@ static int LZ4_compress_generic( size_t refDelta=0; /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ switch(dict) { case noDict: @@ -558,15 +581,15 @@ static int LZ4_compress_generic( BYTE* token; { const BYTE* forwardIp = ip; - unsigned step=1; - unsigned searchMatchNb = (1U << LZ4_skipTrigger); + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; /* Find a match */ do { U32 h = forwardH; ip = forwardIp; forwardIp += step; - step = searchMatchNb++ >> LZ4_skipTrigger; + step = (searchMatchNb++ >> LZ4_skipTrigger); if (unlikely(forwardIp > mflimit)) goto _last_literals; @@ -693,13 +716,22 @@ _next_match: _last_literals: /* Encode Last Literals */ { - int lastRun = (int)(iend - anchor); - if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) + const size_t lastRun = (size_t)(iend - anchor); + if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun<<ML_BITS); - memcpy(op, anchor, iend - anchor); - op += iend-anchor; + if (lastRun >= RUN_MASK) + { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRun<<ML_BITS); + } + memcpy(op, anchor, lastRun); + op += lastRun; } /* End */ @@ -707,19 +739,40 @@ _last_literals: } -int LZ4_compress(const char* source, char* dest, int inputSize) +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + MEM_INIT(state, 0, LZ4_STREAMSIZE); + + if (maxOutputSize >= LZ4_compressBound(inputSize)) + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, 1); + else + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, 1); + } + else + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, 1); + else + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, 1); + } +} + +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_limitedOutput_withState(state, source, dest, inputSize, LZ4_compressBound(inputSize)); +} + +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { #if (HEAPMODE) - void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */ + void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* malloc-calloc aligned on 8-bytes boundaries */ #else U64 ctx[LZ4_STREAMSIZE_U64] = {0}; /* Ensure data is aligned on 8-bytes boundaries */ #endif - int result; - if (inputSize < LZ4_64Klimit) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); - else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); + int result = LZ4_compress_limitedOutput_withState(ctx, source, dest, inputSize, maxOutputSize); #if (HEAPMODE) FREEMEM(ctx); @@ -727,7 +780,13 @@ int LZ4_compress(const char* source, char* dest, int inputSize) return result; } -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +int LZ4_compress(const char* source, char* dest, int inputSize) +{ + return LZ4_compress_limitedOutput(source, dest, inputSize, LZ4_compressBound(inputSize)); +} + + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, unsigned acceleration) { #if (HEAPMODE) void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U64, 8); /* Aligned on 8-bytes boundaries */ @@ -736,10 +795,20 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in #endif int result; - if (inputSize < LZ4_64Klimit) - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); + if (acceleration == 0) + { + if (inputSize < LZ4_64Klimit) + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, ACCELERATION_DEFAULT); + else + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, ACCELERATION_DEFAULT); + } else - result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); + { + if (inputSize < LZ4_64Klimit) + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } #if (HEAPMODE) FREEMEM(ctx); @@ -752,6 +821,14 @@ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, in * Experimental : Streaming functions *****************************************/ +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; +} + /* * LZ4_initStream * Use this function once, to init a newly allocated LZ4_stream_t structure @@ -762,14 +839,6 @@ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } -LZ4_stream_t* LZ4_createStream(void) -{ - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); - LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); - return lz4s; -} - int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { FREEMEM(LZ4_stream); @@ -830,8 +899,7 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) } -FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize, - int maxOutputSize, limitedOutput_directive limit) +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; @@ -858,9 +926,9 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, 1); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, 1); streamPtr->dictSize += (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; @@ -870,9 +938,9 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, 1); else - result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue); + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, 1); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; @@ -882,12 +950,7 @@ FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* so int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited); -} - -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput); + return LZ4_compress_limitedOutput_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize)); } @@ -902,7 +965,7 @@ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); - result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue); + result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; @@ -1325,32 +1388,6 @@ char* LZ4_slideInputBuffer (void* LZ4_Data) return (char*)(ctx->bufferStart + dictSize); } -/* Obsolete compresson functions using User-allocated state */ - -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } - -int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) -{ - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); -} - -int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) -{ - if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ - MEM_INIT(state, 0, LZ4_STREAMSIZE); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); - else - return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); -} - /* Obsolete streaming decompression functions */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) @@ -48,7 +48,7 @@ extern "C" { * Version **************************************/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 6 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); @@ -99,7 +99,7 @@ LZ4_decompress_safe() : * Advanced Functions **************************************/ #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ -#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) /* LZ4_compressBound() : @@ -107,11 +107,11 @@ LZ4_compressBound() : This function is primarily useful for memory allocation purposes (output buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE + inputSize : max supported value is LZ4_MAX_INPUT_SIZE return : maximum output size in a "worst case" scenario or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) */ -int LZ4_compressBound(int isize); +int LZ4_compressBound(int inputSize); /* @@ -130,10 +130,21 @@ int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, /* +LZ4_compress_fast() : + Same as LZ4_compress_limitedOutput, but allows to select an "acceleration" factor. + The larger the value, the faster the algorithm, but also the lesser the compression. + So it's a trade-off, which can be fine tuned, selecting whichever value you want. + An acceleration value of "0" means "use Default value", which is typically about 15 (see lz4.c source code). +*/ +int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxOutputSize, unsigned acceleration); + + +/* LZ4_compress_withState() : Same compression functions, but using an externally allocated memory space to store compression state. Use LZ4_sizeofState() to know how much memory must be allocated, and then, provide it as 'void* state' to compression functions. + Note that 'state' must be aligned on 4-bytes boundaries. */ int LZ4_sizeofState(void); int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); diff --git a/lib/lz4frame.c b/lib/lz4frame.c index 474b196..6c65d0b 100644 --- a/lib/lz4frame.c +++ b/lib/lz4frame.c @@ -204,7 +204,7 @@ static U64 LZ4F_readLE64 (const BYTE* srcPtr) U64 value64 = srcPtr[0]; value64 += (srcPtr[1]<<8); value64 += (srcPtr[2]<<16); - value64 += (srcPtr[3]<<24); + value64 += ((U64)srcPtr[3]<<24); value64 += ((U64)srcPtr[4]<<32); value64 += ((U64)srcPtr[5]<<40); value64 += ((U64)srcPtr[6]<<48); diff --git a/lz4_block_format.txt b/lz4_Block_format.md index e248fd9..b933a6a 100644 --- a/lz4_block_format.txt +++ b/lz4_Block_format.md @@ -1,10 +1,9 @@ LZ4 Block Format Description ============================ -Last revised: 2015-03-26; +Last revised: 2015-03-26. Author : Yann Collet - This small specification intents to provide enough information to anyone willing to produce LZ4-compatible compressed data blocks using any programming language. @@ -26,7 +25,8 @@ on implementation details of the compressor, and vice versa. Compressed block format ----------------------- An LZ4 compressed block is composed of sequences. -Schematically, a sequence is a suite of literals, followed by a match copy. +A sequence is a suite of literals (not-compressed bytes), +followed by a match copy. Each sequence starts with a token. The token is a one byte value, separated into two 4-bits fields. @@ -35,14 +35,14 @@ Therefore each field ranges from 0 to 15. The first field uses the 4 high-bits of the token. It provides the length of literals to follow. -(Note : a literal is a not-compressed byte). + If the field value is 0, then there is no literal. If it is 15, then we need to add some more bytes to indicate the full length. -Each additionnal byte then represent a value from 0 to 255, +Each additional byte then represent a value from 0 to 255, which is added to the previous value to produce a total length. When the byte value is 255, another byte is output. There can be any number of bytes following the token. There is no "size limit". -(Sidenote this is why a not-compressible input block is expanded by 0.4%). +(Side note : this is why a not-compressible input block is expanded by 0.4%). Example 1 : A length of 48 will be represented as : - 15 : value for the 4-bits High field @@ -65,7 +65,8 @@ It's possible that there are zero literal. Following the literals is the match copy operation. It starts by the offset. -This is a 2 bytes value, in little endian format. +This is a 2 bytes value, in little endian format +(the 1st byte is the "low" byte, the 2nd one is the "high" byte). The offset represents the position of the match to be copied from. 1 means "current position - 1 byte". @@ -95,9 +96,12 @@ Parsing restrictions ----------------------- There are specific parsing rules to respect in order to remain compatible with assumptions made by the decoder : -1) The last 5 bytes are always literals -2) The last match must start at least 12 bytes before end of block -Consequently, a block with less than 13 bytes cannot be compressed. + +1. The last 5 bytes are always literals +2. The last match must start at least 12 bytes before end of block. + + Consequently, a block with less than 13 bytes cannot be compressed. + These rules are in place to ensure that the decoder will never read beyond the input buffer, nor write beyond the output buffer. @@ -118,4 +122,3 @@ or full optimal parsing. All these trade-off offer distinctive speed/memory/compression advantages. Whatever the method used by the compressor, its result will be decodable by any LZ4 decoder if it follows the format specification described above. - diff --git a/lz4_Frame_format.md b/lz4_Frame_format.md new file mode 100644 index 0000000..55edaa0 --- /dev/null +++ b/lz4_Frame_format.md @@ -0,0 +1,385 @@ +LZ4 Frame Format Description +============================ + +###Notices + +Copyright (c) 2013-2015 Yann Collet + +Permission is granted to copy and distribute this document +for any purpose and without charge, +including translations into other languages +and incorporation into compilations, +provided that the copyright notice and this notice are preserved, +and that any substantive changes or deletions from the original +are clearly marked. +Distribution of this document is unlimited. + +###Version + +1.5.1 (31/03/2015) + + +Introduction +------------ + +The purpose of this document is to define a lossless compressed data format, +that is independent of CPU type, operating system, +file system and character set, suitable for +File compression, Pipe and streaming compression +using the [LZ4 algorithm](http://www.lz4.info). + +The data can be produced or consumed, +even for an arbitrarily long sequentially presented input data stream, +using only an a priori bounded amount of intermediate storage, +and hence can be used in data communications. +The format uses the LZ4 compression method, +and optional [xxHash-32 checksum method](https://github.com/Cyan4973/xxHash), +for detection of data corruption. + +The data format defined by this specification +does not attempt to allow random access to compressed data. + +This specification is intended for use by implementers of software +to compress data into LZ4 format and/or decompress data from LZ4 format. +The text of the specification assumes a basic background in programming +at the level of bits and other primitive data representations. + +Unless otherwise indicated below, +a compliant compressor must produce data sets +that conform to the specifications presented here. +It doesn’t need to support all options though. + +A compliant decompressor must be able to decompress +at least one working set of parameters +that conforms to the specifications presented here. +It may also ignore checksums. +Whenever it does not support a specific parameter within the compressed stream, +it must produce a non-ambiguous error code +and associated error message explaining which parameter is unsupported. + + +General Structure of LZ4 Frame format +------------------------------------- + +| MagicNb | F. Descriptor | Block | (...) | EndMark | C. Checksum | +|:-------:|:-------------:| ----- | ----- | ------- | ----------- | +| 4 bytes | 3-11 bytes | | | 4 bytes | 4 bytes | + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184D2204 + +__Frame Descriptor__ + +3 to 11 Bytes, to be detailed in the next part. +Most important part of the spec. + +__Data Blocks__ + +To be detailed later on. +That’s where compressed data is stored. + +__EndMark__ + +The flow of blocks ends when the last data block has a size of “0”. +The size is expressed as a 32-bits value. + +__Content Checksum__ + +Content Checksum verify that the full content has been decoded correctly. +The content checksum is the result +of [xxh32() hash function](https://github.com/Cyan4973/xxHash) +digesting the original (decoded) data as input, and a seed of zero. +Content checksum is only present when its associated flag +is set in the frame descriptor. +Content Checksum validates the result, +that all blocks were fully transmitted in the correct order and without error, +and also that the encoding/decoding process itself generated no distortion. +Its usage is recommended. + +__Frame Concatenation__ + +In some circumstances, it may be preferable to append multiple frames, +for example in order to add new data to an existing compressed file +without re-framing it. + +In such case, each frame has its own set of descriptor flags. +Each frame is considered independent. +The only relation between frames is their sequential order. + +The ability to decode multiple concatenated frames +within a single stream or file +is left outside of this specification. +As an example, the reference lz4 command line utility behavior is +to decode all concatenated frames in their sequential order. + + +Frame Descriptor +---------------- + +| FLG | BD | (Content Size) | HC | +| ------- | ------- |:--------------:| ------- | +| 1 byte | 1 byte | 0 - 8 bytes | 1 byte | + +The descriptor uses a minimum of 3 bytes, +and up to 11 bytes depending on optional parameters. + +__FLG byte__ + +| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 | +| ------- | ------- | ------- | --------- | ------- | --------- | -------- | +|FieldName| Version | B.Indep | B.Checksum| C.Size | C.Checksum|*Reserved*| + + +__BD byte__ + +| BitNb | 7 | 6-5-4 | 3-2-1-0 | +| ------- | -------- | ------------ | -------- | +|FieldName|*Reserved*| Block MaxSize|*Reserved*| + +In the tables, bit 7 is highest bit, while bit 0 is lowest. + +__Version Number__ + +2-bits field, must be set to “01”. +Any other value cannot be decoded by this version of the specification. +Other version numbers will use different flag layouts. + +__Block Independence flag__ + +If this flag is set to “1”, blocks are independent. +If this flag is set to “0”, each block depends on previous ones +(up to LZ4 window size, which is 64 KB). +In such case, it’s necessary to decode all blocks in sequence. + +Block dependency improves compression ratio, especially for small blocks. +On the other hand, it makes direct jumps or multi-threaded decoding impossible. + +__Block checksum flag__ + +If this flag is set, each data block will be followed by a 4-bytes checksum, +calculated by using the xxHash-32 algorithm on the raw (compressed) data block. +The intention is to detect data corruption (storage or transmission errors) +immediately, before decoding. +Block checksum usage is optional. + +__Content Size flag__ + +If this flag is set, the uncompressed size of data included within the frame +will be present as an 8 bytes unsigned little endian value, after the flags. +Content Size usage is optional. + +__Content checksum flag__ + +If this flag is set, a content checksum will be appended after the EndMark. + +Recommended value : “1” (content checksum is present) + +__Block Maximum Size__ + +This information is intended to help the decoder allocate memory. +Size here refers to the original (uncompressed) data size. +Block Maximum Size is one value among the following table : + +| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | +| --- | --- | --- | --- | ----- | ------ | ---- | ---- | +| N/A | N/A | N/A | N/A | 64 KB | 256 KB | 1 MB | 4 MB | + +The decoder may refuse to allocate block sizes above a (system-specific) size. +Unused values may be used in a future revision of the spec. +A decoder conformant to the current version of the spec +is only able to decode blocksizes defined in this spec. + +__Reserved bits__ + +Value of reserved bits **must** be 0 (zero). +Reserved bit might be used in a future version of the specification, +typically enabling new optional features. +If this happens, a decoder respecting the current version of the specification +shall not be able to decode such a frame. + +__Content Size__ + +This is the original (uncompressed) size. +This information is optional, and only present if the associated flag is set. +Content size is provided using unsigned 8 Bytes, for a maximum of 16 HexaBytes. +Format is Little endian. +This value is informational. +It can be safely skipped by a conformant decoder. + +__Header Checksum__ + +One-byte checksum of combined descriptor fields, including optional ones. +The value is the second byte of xxh32() : ` (xxh32()>>8) & 0xFF } ` +using zero as a seed, +and the full Frame Descriptor as an input +(including optional fields when they are present). +A wrong checksum indicates an error in the descriptor. +Header checksum is informational and can be skipped. + + +Data Blocks +----------- + +| Block Size | data | (Block Checksum) | +|:----------:| ------ |:----------------:| +| 4 bytes | | 0 - 4 bytes | + + +__Block Size__ + +This field uses 4-bytes, format is little-endian. + +The highest bit is “1” if data in the block is uncompressed. + +The highest bit is “0” if data in the block is compressed by LZ4. + +All other bits give the size, in bytes, of the following data block +(the size does not include the block checksum if present). + +Block Size shall never be larger than Block Maximum Size. +Such a thing could happen for incompressible source data. +In such case, such a data block shall be passed in uncompressed format. + +__Data__ + +Where the actual data to decode stands. +It might be compressed or not, depending on previous field indications. +Uncompressed size of Data can be any size, up to “block maximum size”. +Note that data block is not necessarily full : +an arbitrary “flush” may happen anytime. Any block can be “partially filled”. + +__Block checksum__ + +Only present if the associated flag is set. +This is a 4-bytes checksum value, in little endian format, +calculated by using the xxHash-32 algorithm on the raw (undecoded) data block, +and a seed of zero. +The intention is to detect data corruption (storage or transmission errors) +before decoding. + +Block checksum is cumulative with Content checksum. + + +Skippable Frames +---------------- + +| Magic Number | Frame Size | User Data | +|:------------:|:----------:| --------- | +| 4 bytes | 4 bytes | | + +Skippable frames allow the integration of user-defined data +into a flow of concatenated frames. +Its design is pretty straightforward, +with the sole objective to allow the decoder to quickly skip +over user-defined data and continue decoding. + +For the purpose of facilitating identification, +it is discouraged to start a flow of concatenated frames with a skippable frame. +If there is a need to start such a flow with some user data +encapsulated into a skippable frame, +it’s recommended to start with a zero-byte LZ4 frame +followed by a skippable frame. +This will make it easier for file type identifiers. + + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184D2A5X, which means any value from 0x184D2A50 to 0x184D2A5F. +All 16 values are valid to identify a skippable frame. + +__Frame Size__ + +This is the size, in bytes, of the following User Data +(without including the magic number nor the size field itself). +4 Bytes, Little endian format, unsigned 32-bits. +This means User Data can’t be bigger than (2^32-1) Bytes. + +__User Data__ + +User Data can be anything. Data will just be skipped by the decoder. + + +Legacy frame +------------ + +The Legacy frame format was defined into the initial versions of “LZ4Demo”. +Newer compressors should not use this format anymore, as it is too restrictive. + +Main characteristics of the legacy format : + +- Fixed block size : 8 MB. +- All blocks must be completely filled, except the last one. +- All blocks are always compressed, even when compression is detrimental. +- The last block is detected either because + it is followed by the “EOF” (End of File) mark, + or because it is followed by a known Frame Magic Number. +- No checksum +- Convention is Little endian + +| MagicNb | B.CSize | CData | B.CSize | CData | (...) | EndMark | +| ------- | ------- | ----- | ------- | ----- | ------- | ------- | +| 4 bytes | 4 bytes | CSize | 4 bytes | CSize | x times | EOF | + + +__Magic Number__ + +4 Bytes, Little endian format. +Value : 0x184C2102 + +__Block Compressed Size__ + +This is the size, in bytes, of the following compressed data block. +4 Bytes, Little endian format. + +__Data__ + +Where the actual compressed data stands. +Data is always compressed, even when compression is detrimental. + +__EndMark__ + +End of compressed stream is implicit. +It needs to be followed by a standard EOF (End Of File) signal, +wether it is a file or a stream. + +Alternatively, if the frame is followed by a valid Frame Magic Number, +it is considered completed. +It makes legacy frames compatible with frame concatenation. + +Any other value will be interpreted as a block size, +and trigger an error if it does not fit within acceptable range. + + +Version changes +--------------- + +1.5.1 : changed format to MarkDown compatible + +1.5 : removed Dictionary ID from specification + +1.4.1 : changed wording from “stream” to “frame” + +1.4 : added skippable streams, re-added stream checksum + +1.3 : modified header checksum + +1.2 : reduced choice of “block size”, to postpone decision on “dynamic size of BlockSize Field”. + +1.1 : optional fields are now part of the descriptor + +1.0 : changed “block size” specification, adding a compressed/uncompressed flag + +0.9 : reduced scale of “block maximum size” table + +0.8 : removed : high compression flag + +0.7 : removed : stream checksum + +0.6 : settled : stream size uses 8 bytes, endian convention is little endian + +0.5: added copyright notice + +0.4 : changed format to Google Doc compatible OpenDocument
\ No newline at end of file diff --git a/programs/Makefile b/programs/Makefile index a324148..9a1bade 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -27,11 +27,14 @@ # lz4c32: Same as lz4c, but forced to compile in 32-bits mode # fuzzer : Test tool, to check lz4 integrity on target platform # fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# frametest : Test tool, to check lz4frame integrity on target platform +# frametest32: Same as frametest, but forced to compile in 32-bits mode # fullbench : Precisely measure speed for each LZ4 function variant # fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# datagen : generates synthetic data samples for tests & benchmarks # ########################################################################## -RELEASE?= r128 +RELEASE?= r129 DESTDIR?= PREFIX ?= /usr/local @@ -168,6 +171,11 @@ test-lz4-contentSize: lz4 datagen ./lz4 -v tmp | ./lz4 -t ./lz4 -v --content-size tmp | ./lz4 -d > tmp2 diff -s tmp tmp2 + # test large size [2-4] GB + @./datagen -g3G -P100 | ./lz4 | ./lz4 --decompress --force --sparse - tmp + @ls -ls tmp + ./lz4 --quiet --content-size tmp | ./lz4 --verbose --decompress --force --sparse - tmp2 + @ls -ls tmp2 @rm tmp* test-lz4-frame-concatenation: lz4 datagen diff --git a/programs/fullbench.c b/programs/fullbench.c index 0c6e05e..41075bc 100644 --- a/programs/fullbench.c +++ b/programs/fullbench.c @@ -146,9 +146,9 @@ struct chunkParameters -//************************************** -// Benchmark Parameters -//************************************** +/************************************** +* Benchmark Parameters +**************************************/ static int chunkSize = DEFAULT_CHUNKSIZE; static int nbIterations = NBLOOPS; static int BMK_pause = 0; @@ -175,17 +175,17 @@ void BMK_SetPause(void) BMK_pause = 1; } -//********************************************************* -// Private functions -//********************************************************* +/********************************************************* +* Private functions +*********************************************************/ #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) { - // Based on Legacy ftime() - // Rolls over every ~ 12.1 days (0x100000/24/60/60) - // Use GetMilliSpan to correct for rollover + /* Based on Legacy ftime() + * Rolls over every ~ 12.1 days (0x100000/24/60/60) + * Use GetMilliSpan to correct for rollover */ struct timeb tb; int nCount; ftime( &tb ); @@ -197,8 +197,8 @@ static int BMK_GetMilliStart(void) static int BMK_GetMilliStart(void) { - // Based on newer gettimeofday() - // Use GetMilliSpan to correct for rollover + /* Based on newer gettimeofday() + * Use GetMilliSpan to correct for rollover */ struct timeval tv; int nCount; gettimeofday(&tv, NULL); @@ -383,7 +383,12 @@ start: static int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); +} + +static int local_LZ4_compress_fast(const char* in, char* out, int inSize) +{ + return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0); } static void* stateLZ4; @@ -394,7 +399,7 @@ static int local_LZ4_compress_withState(const char* in, char* out, int inSize) static int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)-1); } static LZ4_stream_t* ctx; @@ -405,7 +410,7 @@ static int local_LZ4_compress_continue(const char* in, char* out, int inSize) static int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) { - return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)-1); } @@ -432,12 +437,12 @@ static int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSiz static int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)-1); } static int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)-1); } static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) @@ -447,7 +452,7 @@ static int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) static int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) { - return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize)); + return LZ4_compressHC_limitedOutput_continue((LZ4_streamHC_t*)ctx, in, out, inSize, LZ4_compressBound(inSize)-1); } static int local_LZ4F_compressFrame(const char* in, char* out, int inSize) @@ -472,7 +477,6 @@ static int local_LZ4_saveDictHC(const char* in, char* out, int inSize) static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) { (void)inSize; - //lz4_decode_sse((BYTE*)out, (BYTE*)in, inSize); LZ4_decompress_fast(in, out, outSize); return outSize; } @@ -530,7 +534,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) { int fileIdx=0; char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 16 +# define NB_COMPRESSION_ALGORITHMS 17 double totalCTime[NB_COMPRESSION_ALGORITHMS+1] = {0}; double totalCSize[NB_COMPRESSION_ALGORITHMS+1] = {0}; # define NB_DECOMPRESSION_ALGORITHMS 9 @@ -544,7 +548,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) return 10; } - // Loop for each file + /* Loop for each fileName */ while (fileIdx<nbFiles) { FILE* inFile; @@ -559,11 +563,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) U32 crcOriginal; - // Init - stateLZ4 = LZ4_createStream(); - stateLZ4HC = LZ4_createStreamHC(); - - // Check file existence + /* Check file existence */ inFileName = fileNamesTable[fileIdx++]; inFile = fopen( inFileName, "rb" ); if (inFile==NULL) @@ -572,18 +572,18 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) return 11; } - // Memory allocation & restrictions + /* Memory size adjustments */ inFileSize = BMK_GetFileSize(inFileName); if (inFileSize==0) { DISPLAY( "file is empty\n"); return 11; } - benchedSize = (size_t) BMK_findMaxMem(inFileSize) / 2; + benchedSize = (size_t) BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */ if (benchedSize==0) { DISPLAY( "not enough memory\n"); return 11; } if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) - { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } - // Alloc + /* Allocation */ + stateLZ4 = LZ4_createStream(); + stateLZ4HC = LZ4_createStreamHC(); chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)chunkSize)+1) * sizeof(struct chunkParameters)); orig_buff = (char*) malloc((size_t)benchedSize); nbChunks = (int) (((int)benchedSize + (chunkSize-1))/ chunkSize); @@ -602,7 +602,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) return 12; } - // Fill input buffer + /* Fill in src buffer */ DISPLAY("Loading %s... \r", inFileName); readSize = fread(orig_buff, 1, benchedSize, inFile); fclose(inFile); @@ -616,11 +616,11 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) return 13; } - // Calculating input Checksum + /* Calculating input Checksum */ crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0); - // Bench + /* Bench */ { int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; size_t cSize=0; @@ -637,7 +637,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) void* (*initFunction)(const char*) = NULL; double bestTime = 100000000.; - // Init data chunks + /* Init data chunks */ { int i; size_t remaining = benchedSize; @@ -664,20 +664,21 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) case 4 : compressionFunction = local_LZ4_compress_limitedOutput_withState; compressorName = "LZ4_compress_limitedOutput_withState"; break; case 5 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_continue"; break; case 6 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; compressorName = "LZ4_compress_limitedOutput_continue"; break; - case 7 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; - case 8 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; - case 9 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; - case 10: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; - case 11: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; - case 12: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; - case 13: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; - case 14: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; + case 7 : compressionFunction = local_LZ4_compress_fast; compressorName = "LZ4_compress_fast"; break; + case 8 : compressionFunction = LZ4_compressHC; compressorName = "LZ4_compressHC"; break; + case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput; compressorName = "LZ4_compressHC_limitedOutput"; break; + case 10 : compressionFunction = local_LZ4_compressHC_withStateHC; compressorName = "LZ4_compressHC_withStateHC"; break; + case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; compressorName = "LZ4_compressHC_limitedOutput_withStateHC"; break; + case 12: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_continue"; break; + case 13: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; compressorName = "LZ4_compressHC_limitedOutput_continue"; break; + case 14: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break; + case 15: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame"; chunkP[0].origSize = (int)benchedSize; nbChunks=1; break; - case 15: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; + case 16: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict"; LZ4_loadDict(&LZ4_dict, chunkP[0].origBuffer, chunkP[0].origSize); break; - case 16: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; + case 17: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC"; LZ4_loadDictHC(&LZ4_dictHC, chunkP[0].origBuffer, chunkP[0].origSize); break; default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; @@ -689,7 +690,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) int milliTime; PROGRESS("%1i- %-28.28s :%9i ->\r", loopNb, compressorName, (int)benchedSize); - { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } // warming up memory + { size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */ nb_loops = 0; milliTime = BMK_GetMilliStart(); @@ -724,8 +725,8 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) totalCSize[cAlgNb] += cSize; } - // Prepare layout for decompression - // Init data chunks + /* Prepare layout for decompression */ + /* Init data chunks */ { int i; size_t remaining = benchedSize; @@ -747,7 +748,7 @@ int fullSpeedBench(char** fileNamesTable, int nbFiles) if (chunkP[chunkNb].compressedSize==0) DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress"), exit(1); } - // Decompression Algorithms + /* Decompression Algorithms */ for (dAlgNb=1; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); dAlgNb++) { //const char* dName = decompressionNames[dAlgNb]; @@ -940,7 +941,7 @@ _exit_blockProperties: // Modify Nb Iterations case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) + if ((argument[1] >='0') && (argument[1] <='9')) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); diff --git a/programs/lz4cli.c b/programs/lz4cli.c index da5da71..888f21e 100644 --- a/programs/lz4cli.c +++ b/programs/lz4cli.c @@ -65,7 +65,7 @@ /**************************** * OS-specific Includes *****************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) # include <io.h> /* _isatty */ # ifdef __MINGW32__ int _fileno(FILE *stream); /* MINGW somehow forgets to include this prototype into <stdio.h> */ diff --git a/programs/lz4io.c b/programs/lz4io.c index 02e03c8..0a8132a 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -61,7 +61,7 @@ /****************************** * OS-specific Includes ******************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) # include <fcntl.h> /* _O_BINARY */ # include <io.h> /* _setmode, _fileno, _get_osfhandle */ # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) @@ -412,7 +412,6 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena /* Init */ start = clock(); memset(&prefs, 0, sizeof(prefs)); - if ((g_displayLevel==2) && (compressionLevel>=3)) g_displayLevel=3; errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode)); get_fileHandle(input_filename, output_filename, &finput, &foutput); @@ -428,6 +427,8 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena { unsigned long long fileSize = LZ4IO_GetFileSize(input_filename); prefs.frameInfo.contentSize = fileSize; /* == 0 if input == stdin */ + if (fileSize==0) + DISPLAYLEVEL(3, "Warning : cannot determine uncompressed frame content size \n"); } /* Allocate Memory */ @@ -456,7 +457,7 @@ int LZ4IO_compressFilename(const char* input_filename, const char* output_filena outSize = LZ4F_compressUpdate(ctx, out_buff, outBuffSize, in_buff, readSize, NULL); if (LZ4F_isError(outSize)) EXM_THROW(34, "Compression failed : %s", LZ4F_getErrorName(outSize)); compressedfilesize += outSize; - DISPLAYUPDATE(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + DISPLAYUPDATE(2, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); /* Write Block */ sizeCheck = fwrite(out_buff, 1, outSize, foutput); @@ -639,6 +640,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) { /* Write Block */ filesize += decodedBytes; + DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); if (g_sparseFileSupport) { size_t* const oBuffStartT = (size_t*)outBuff; /* since outBuff is malloc'ed, it's aligned on size_t */ diff --git a/visual/2012/datagen/datagen.vcxproj b/visual/2012/datagen/datagen.vcxproj new file mode 100644 index 0000000..ea61533 --- /dev/null +++ b/visual/2012/datagen/datagen.vcxproj @@ -0,0 +1,159 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{D745AE2F-596A-403A-9B91-81A8C6779243}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>datagen</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\programs\datagen.c" />
+ <ClCompile Include="..\..\..\programs\datagencli.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\programs\datagen.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/datagen/datagen.vcxproj.filters b/visual/2012/datagen/datagen.vcxproj.filters new file mode 100644 index 0000000..822d9bc --- /dev/null +++ b/visual/2012/datagen/datagen.vcxproj.filters @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Fichiers sources">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Fichiers d%27en-tête">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Fichiers de ressources">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\programs\datagen.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\programs\datagencli.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\programs\datagen.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ </ItemGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/frametest/frametest.vcxproj b/visual/2012/frametest/frametest.vcxproj new file mode 100644 index 0000000..b86f7de --- /dev/null +++ b/visual/2012/frametest/frametest.vcxproj @@ -0,0 +1,166 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>frametest</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\programs\frametest.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/frametest/frametest.vcxproj.filters b/visual/2012/frametest/frametest.vcxproj.filters new file mode 100644 index 0000000..19c3d15 --- /dev/null +++ b/visual/2012/frametest/frametest.vcxproj.filters @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Fichiers sources">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Fichiers d%27en-tête">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Fichiers de ressources">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4frame.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4hc.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\xxhash.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\programs\frametest.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4hc.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\xxhash.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ </ItemGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/fullbench/fullbench.vcxproj b/visual/2012/fullbench/fullbench.vcxproj new file mode 100644 index 0000000..11f8f45 --- /dev/null +++ b/visual/2012/fullbench/fullbench.vcxproj @@ -0,0 +1,166 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>fullbench</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v110</PlatformToolset>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ <IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <WarningLevel>Level4</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c" />
+ <ClCompile Include="..\..\..\lib\lz4frame.c" />
+ <ClCompile Include="..\..\..\lib\lz4hc.c" />
+ <ClCompile Include="..\..\..\lib\xxhash.c" />
+ <ClCompile Include="..\..\..\programs\fullbench.c" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame.h" />
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h" />
+ <ClInclude Include="..\..\..\lib\lz4hc.h" />
+ <ClInclude Include="..\..\..\lib\xxhash.h" />
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/fullbench/fullbench.vcxproj.filters b/visual/2012/fullbench/fullbench.vcxproj.filters new file mode 100644 index 0000000..cdcf599 --- /dev/null +++ b/visual/2012/fullbench/fullbench.vcxproj.filters @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <Filter Include="Fichiers sources">
+ <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+ <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+ </Filter>
+ <Filter Include="Fichiers d%27en-tête">
+ <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
+ <Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
+ </Filter>
+ <Filter Include="Fichiers de ressources">
+ <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
+ <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
+ </Filter>
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="..\..\..\lib\lz4.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4frame.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\lz4hc.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\lib\xxhash.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\..\programs\fullbench.c">
+ <Filter>Fichiers sources</Filter>
+ </ClCompile>
+ </ItemGroup>
+ <ItemGroup>
+ <ClInclude Include="..\..\..\lib\lz4.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4frame_static.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\lz4hc.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\..\lib\xxhash.h">
+ <Filter>Fichiers d%27en-tête</Filter>
+ </ClInclude>
+ </ItemGroup>
+</Project>
\ No newline at end of file diff --git a/visual/2012/fuzzer/fuzzer.vcxproj b/visual/2012/fuzzer/fuzzer.vcxproj index 9ba4fab..4da822f 100644 --- a/visual/2012/fuzzer/fuzzer.vcxproj +++ b/visual/2012/fuzzer/fuzzer.vcxproj @@ -89,6 +89,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -102,6 +103,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -117,6 +119,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -134,6 +137,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
diff --git a/visual/2012/fuzzer/fuzzer.vcxproj.user b/visual/2012/fuzzer/fuzzer.vcxproj.user deleted file mode 100644 index 7cbb321..0000000 --- a/visual/2012/fuzzer/fuzzer.vcxproj.user +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <PropertyGroup />
-</Project>
\ No newline at end of file diff --git a/visual/2012/lz4.sln b/visual/2012/lz4.sln index 2d07075..80d1a54 100644 --- a/visual/2012/lz4.sln +++ b/visual/2012/lz4.sln @@ -5,40 +5,60 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "lz4", "lz4\lz4.vcxproj", "{ EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcxproj", "{18B9F1A7-9C66-4352-898B-30804DADE0FD}"
EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "frametest", "frametest\frametest.vcxproj", "{39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{D745AE2F-596A-403A-9B91-81A8C6779243}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
- Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
- Release|Mixed Platforms = Release|Mixed Platforms
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Mixed Platforms.Build.0 = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.ActiveCfg = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|Win32.Build.0 = Debug|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.ActiveCfg = Debug|x64
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Debug|x64.Build.0 = Debug|x64
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.ActiveCfg = Release|Win32
- {E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Mixed Platforms.Build.0 = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.ActiveCfg = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|Win32.Build.0 = Release|Win32
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.ActiveCfg = Release|x64
{E30329AC-0057-4FE0-8FDA-7F650D398C4C}.Release|x64.Build.0 = Release|x64
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Mixed Platforms.Build.0 = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.ActiveCfg = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|Win32.Build.0 = Debug|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.ActiveCfg = Debug|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Debug|x64.Build.0 = Debug|x64
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.ActiveCfg = Release|Win32
- {18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Mixed Platforms.Build.0 = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.ActiveCfg = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|Win32.Build.0 = Release|Win32
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.ActiveCfg = Release|x64
{18B9F1A7-9C66-4352-898B-30804DADE0FD}.Release|x64.Build.0 = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.ActiveCfg = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|Win32.Build.0 = Debug|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.ActiveCfg = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Debug|x64.Build.0 = Debug|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.ActiveCfg = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|Win32.Build.0 = Release|Win32
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.ActiveCfg = Release|x64
+ {6A4DF4EF-C77F-43C6-8901-DDCD20879E4E}.Release|x64.Build.0 = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.ActiveCfg = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|Win32.Build.0 = Debug|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.ActiveCfg = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Debug|x64.Build.0 = Debug|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.ActiveCfg = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|Win32.Build.0 = Release|Win32
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.ActiveCfg = Release|x64
+ {39AD6ECC-8BAD-4368-95E4-A1AA2F077BB7}.Release|x64.Build.0 = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.ActiveCfg = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|Win32.Build.0 = Debug|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.ActiveCfg = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Debug|x64.Build.0 = Debug|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.ActiveCfg = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|Win32.Build.0 = Release|Win32
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.ActiveCfg = Release|x64
+ {D745AE2F-596A-403A-9B91-81A8C6779243}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
diff --git a/visual/2012/lz4.v11.suo b/visual/2012/lz4.v11.suo Binary files differdeleted file mode 100644 index 8321984..0000000 --- a/visual/2012/lz4.v11.suo +++ /dev/null diff --git a/visual/2012/lz4/lz4.vcxproj b/visual/2012/lz4/lz4.vcxproj index 5540d5d..8ac9387 100644 --- a/visual/2012/lz4/lz4.vcxproj +++ b/visual/2012/lz4/lz4.vcxproj @@ -89,6 +89,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -102,6 +103,7 @@ <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -117,6 +119,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -134,6 +137,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
diff --git a/visual/2012/lz4/lz4.vcxproj.user b/visual/2012/lz4/lz4.vcxproj.user deleted file mode 100644 index 7cbb321..0000000 --- a/visual/2012/lz4/lz4.vcxproj.user +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
- <PropertyGroup />
-</Project>
\ No newline at end of file |
