From 648678788c751c0d89737dbb8b3144fa9d2c1592 Mon Sep 17 00:00:00 2001 From: "yann.collet.73@gmail.com" Date: Tue, 7 Jan 2014 18:47:50 +0000 Subject: Makefile : added capability to install libraries Modified Directory tree, to better separate libraries from programs. git-svn-id: https://lz4.googlecode.com/svn/trunk@111 650e7d94-2a16-8b24-b05c-7c0b3f6821cd --- COPYING | 339 ----------- LICENSE | 24 + Makefile | 144 ++--- NEWS | 4 + bench.c | 441 -------------- bench.h | 41 -- cmake/CMakeLists.txt | 80 --- cmake/pack/CMakeLists.txt | 82 --- cmake/pack/release_COPYING.txt | 21 - cmake_unofficial/CMakeLists.txt | 81 +++ fullbench.c | 742 ----------------------- fuzzer.c | 303 ---------- lz4.1 | 87 --- lz4_format_description.txt | 240 ++++---- lz4cli.c | 1257 --------------------------------------- programs/COPYING | 339 +++++++++++ programs/Makefile | 100 ++++ programs/bench.c | 441 ++++++++++++++ programs/bench.h | 41 ++ programs/fullbench.c | 742 +++++++++++++++++++++++ programs/fuzzer.c | 303 ++++++++++ programs/lz4.1 | 87 +++ programs/lz4cli.c | 1257 +++++++++++++++++++++++++++++++++++++++ programs/xxhash.c | 475 +++++++++++++++ programs/xxhash.h | 164 +++++ xxhash.c | 475 --------------- xxhash.h | 164 ----- 27 files changed, 4250 insertions(+), 4224 deletions(-) delete mode 100644 COPYING create mode 100644 LICENSE delete mode 100644 bench.c delete mode 100644 bench.h delete mode 100644 cmake/CMakeLists.txt delete mode 100644 cmake/pack/CMakeLists.txt delete mode 100644 cmake/pack/release_COPYING.txt create mode 100644 cmake_unofficial/CMakeLists.txt delete mode 100644 fullbench.c delete mode 100644 fuzzer.c delete mode 100644 lz4.1 delete mode 100644 lz4cli.c create mode 100644 programs/COPYING create mode 100644 programs/Makefile create mode 100644 programs/bench.c create mode 100644 programs/bench.h create mode 100644 programs/fullbench.c create mode 100644 programs/fuzzer.c create mode 100644 programs/lz4.1 create mode 100644 programs/lz4cli.c create mode 100644 programs/xxhash.c create mode 100644 programs/xxhash.h delete mode 100644 xxhash.c delete mode 100644 xxhash.h diff --git a/COPYING b/COPYING deleted file mode 100644 index d159169..0000000 --- a/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..04403e4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +LZ4 Library +Copyright (c) 2013, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/Makefile b/Makefile index 7625941..fd2b9be 100644 --- a/Makefile +++ b/Makefile @@ -1,110 +1,110 @@ # ################################################################ -# LZ4 Makefile -# Copyright (C) Yann Collet 2011-2013 -# GPL v2 License -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# +# LZ4 library - Makefile +# Copyright (C) Yann Collet 2011-2014 +# All rights reserved. +# +# BSD license +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, this +# list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# # You can contact the author at : # - LZ4 source repository : http://code.google.com/p/lz4/ # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c # ################################################################ -# lz4 : Command Line Utility, supporting gzip-like arguments -# lz4c : CLU, supporting also legacy lz4demo arguments -# lz4c32: Same as lz4c, but forced to compile in 32-bits mode -# fuzzer : Test tool, to check lz4 integrity on target platform -# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode -# fullbench : Precisely measure speed for each LZ4 function variant -# fullbench32: Same as fullbench, but forced to compile in 32-bits mode -# ################################################################ -RELEASE=r110 +RELEASE=r111 DESTDIR= -PREFIX=${DESTDIR}/usr -BINDIR=$(PREFIX)/bin -MANDIR=$(PREFIX)/share/man/man1 -DISTRIBNAME=lz4-$(RELEASE).tar.gz +PREFIX=/usr CC=gcc -CFLAGS=-I. -std=c99 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\" +CFLAGS+= -I. -std=c99 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\" + +LIBDIR=$(PREFIX)/lib +INCLUDEDIR=$(PREFIX)/include +PRGDIR=programs +DISTRIBNAME=lz4-$(RELEASE).tar.gz + # Define *.exe as extension for Windows systems -# ifeq ($(OS),Windows_NT) ifneq (,$(filter Windows%,$(OS))) EXT =.exe else EXT = endif -TEXT = bench.c bench.h fullbench.c fuzzer.c lz4.1 lz4.c lz4cli.c \ - lz4_format_description.txt lz4.h lz4hc.c lz4hc.h \ - Makefile xxhash.c xxhash.h \ - NEWS COPYING \ - cmake/CMakeLists.txt cmake/pack/release_COPYING.txt \ - cmake/pack/CMakeLists.txt +TEXT = lz4.c lz4.h lz4hc.c lz4hc.h \ + lz4_format_description.txt Makefile NEWS LICENSE \ + cmake_unofficial/CMakeLists.txt \ + $(PRGDIR)/fullbench.c $(PRGDIR)/fuzzer.c $(PRGDIR)/lz4cli.c \ + $(PRGDIR)/bench.c $(PRGDIR)/bench.h \ + $(PRGDIR)/xxhash.c $(PRGDIR)/xxhash.h \ + $(PRGDIR)/lz4.1 $(PRGDIR)/Makefile $(PRGDIR)/COPYING NONTEXT = LZ4_Streaming_Format.odt SOURCES = $(TEXT) $(NONTEXT) -default: lz4 lz4c - -all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32 - -lz4: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c - $(CC) -O3 $(CFLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) - -lz4c : lz4.c lz4hc.c bench.c xxhash.c lz4cli.c - $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) +default: liblz4 -lz4c32: lz4.c lz4hc.c bench.c xxhash.c lz4cli.c - $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) +all: liblz4 lz4programs -fuzzer : lz4.c lz4hc.c fuzzer.c - @echo fuzzer is a test tool to check lz4 integrity on target platform - $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) +liblz4: liblz4.a liblz4.so -fuzzer32: lz4.c lz4hc.c fuzzer.c - $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) +lz4programs: lz4.c lz4hc.c + @cd $(PRGDIR); make all -fullbench : lz4.c lz4hc.c xxhash.c fullbench.c - $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) +liblz4.a: lz4.c lz4hc.c + $(CC) -O3 -c $(CFLAGS) $^ + ar rcs liblz4.a lz4.o lz4hc.o -fullbench32: lz4.c lz4hc.c xxhash.c fullbench.c - $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) +liblz4.so: lz4.c lz4hc.c lz4.h + $(CC) -shared -fPIC -Wl,--soname=liblz4.so.1 $(CFLAGS) -o $@ clean: - @rm -f core *.o lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) \ - $(DISTRIBNAME) + @rm -f core *.o *.a *.so $(DISTRIBNAME) + @cd $(PRGDIR); make clean @echo Cleaning completed -ifeq ($(shell uname),Linux) +#ifeq ($(shell uname),Linux) +ifneq (,$(filter $(shell uname),Linux Darwin)) -install: lz4 lz4c - @install -d -m 755 $(BINDIR)/ $(MANDIR)/ - @install -m 755 lz4 $(BINDIR)/lz4 - @install -m 755 lz4c $(BINDIR)/lz4c - @install -m 644 lz4.1 $(MANDIR)/lz4.1 +install: liblz4 + @install -d -m 755 $(DESTDIR)$(LIBDIR)/ $(DESTDIR)$(INCLUDEDIR)/ + @install -m 755 liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a + @install -m 755 liblz4.so $(DESTDIR)$(LIBDIR)/liblz4.so + @install -m 755 lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + @install -m 755 lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @echo lz4 static and shared library installed + @cd $(PRGDIR); make install uninstall: - [ -x $(BINDIR)/lz4 ] && rm -f $(BINDIR)/lz4 - [ -x $(BINDIR)/lz4c ] && rm -f $(BINDIR)/lz4c - [ -f $(MANDIR)/lz4.1 ] && rm -f $(MANDIR)/lz4.1 + [ -x $(DESTDIR)$(LIBDIR)/liblz4.a ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.a + [ -x $(DESTDIR)$(LIBDIR)/liblz4.so ] && rm -f $(DESTDIR)$(LIBDIR)/liblz4.so + [ -f $(DESTDIR)$(INCLUDEDIR)/lz4.h ] && rm -f lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h + [ -f $(DESTDIR)$(INCLUDEDIR)/lz4hc.h ] && rm -f lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h + @echo lz4 libraries successfully uninstalled + @cd $(PRGDIR); make uninstall dist: clean - @install -dD -m 700 lz4-$(RELEASE)/cmake/pack/ + @install -dD -m 700 lz4-$(RELEASE)/programs/ + @install -dD -m 700 lz4-$(RELEASE)/cmake_unofficial/ @for f in $(TEXT); do \ tr -d '\r' < $$f > .tmp; \ install -m 600 .tmp lz4-$(RELEASE)/$$f; \ diff --git a/NEWS b/NEWS index 40a9377..8090f71 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +r111 : +Makefile : added capability to install libraries +Modified Directory tree, to better separate libraries from programs. + r110 : lz4 & lz4hc : added capability to allocate state & stream state with custom allocator (issue 99) fuzzer & fullbench : updated to test new functions diff --git a/bench.c b/bench.c deleted file mode 100644 index df3c44a..0000000 --- a/bench.c +++ /dev/null @@ -1,441 +0,0 @@ -/* - bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2013 - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ - -//************************************** -// Compiler Options -//************************************** -// Disable some Visual warning messages -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE // VS2005 - -// Unix Large Files support (>4GB) -#define _FILE_OFFSET_BITS 64 -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif - -// S_ISREG & gettimeofday() are not supported by MSVC -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - - -//************************************** -// Includes -//************************************** -#include // malloc -#include // fprintf, fopen, ftello64 -#include // stat64 -#include // stat64 - -// Use ftime() if gettimeofday() is not available on your target -#if defined(BMK_LEGACY_TIMER) -# include // timeb, ftime -#else -# include // gettimeofday -#endif - -#include "lz4.h" -#define COMPRESSOR0 LZ4_compress -#include "lz4hc.h" -#define COMPRESSOR1 LZ4_compressHC -#define DEFAULTCOMPRESSOR COMPRESSOR0 - -#include "xxhash.h" - - -//************************************** -// Compiler specifics -//************************************** -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - -// GCC does not support _rotl outside of Windows -#if !defined(_WIN32) -# define _rotl(x,r) ((x << r) | (x >> (32 - r))) -#endif - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -//************************************** -// Constants -//************************************** -#define NBLOOPS 3 -#define TIMELOOP 2000 - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -#define KNUTH 2654435761U -#define MAX_MEM (2 GB - 64 MB) -#define DEFAULT_CHUNKSIZE (4 MB) - - -//************************************** -// Local structures -//************************************** -struct chunkParameters -{ - U32 id; - char* origBuffer; - char* compressedBuffer; - int origSize; - int compressedSize; -}; - -struct compressionParameters -{ - int (*compressionFunction)(const char*, char*, int); - int (*decompressionFunction)(const char*, char*, int); -}; - - -//************************************** -// MACRO -//************************************** -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) - - - -//************************************** -// Benchmark Parameters -//************************************** -static int chunkSize = DEFAULT_CHUNKSIZE; -static int nbIterations = NBLOOPS; -static int BMK_pause = 0; - -void BMK_SetBlocksize(int bsize) { chunkSize = bsize; } - -void BMK_SetNbIterations(int nbLoops) -{ - nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", nbIterations); -} - -void BMK_SetPause() { BMK_pause = 1; } - - -//********************************************************* -// Private functions -//********************************************************* - -#if defined(BMK_LEGACY_TIMER) - -static int BMK_GetMilliStart() -{ - // Based on Legacy ftime() - // Rolls over every ~ 12.1 days (0x100000/24/60/60) - // Use GetMilliSpan to correct for rollover - struct timeb tb; - int nCount; - ftime( &tb ); - nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); - return nCount; -} - -#else - -static int BMK_GetMilliStart() -{ - // Based on newer gettimeofday() - // Use GetMilliSpan to correct for rollover - struct timeval tv; - int nCount; - gettimeofday(&tv, NULL); - nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); - return nCount; -} - -#endif - - -static int BMK_GetMilliSpan( int nTimeStart ) -{ - int nSpan = BMK_GetMilliStart() - nTimeStart; - if ( nSpan < 0 ) - nSpan += 0x100000 * 1000; - return nSpan; -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t step = (64 MB); - BYTE* testmem=NULL; - - requiredMem = (((requiredMem >> 26) + 1) << 26); - requiredMem += 2*step; - if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - - while (!testmem) - { - requiredMem -= step; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } - - free (testmem); - return (size_t) (requiredMem - step); -} - - -static U64 BMK_GetFileSize(char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... - return (U64)statbuf.st_size; -} - - -//********************************************************* -// Public function -//********************************************************* - -int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) -{ - int fileIdx=0; - char* orig_buff; - struct compressionParameters compP; - int cfunctionId; - - U64 totals = 0; - U64 totalz = 0; - double totalc = 0.; - double totald = 0.; - - - // Init - if (cLevel <3) cfunctionId = 0; else cfunctionId = 1; - switch (cfunctionId) - { -#ifdef COMPRESSOR0 - case 0 : compP.compressionFunction = COMPRESSOR0; break; -#endif -#ifdef COMPRESSOR1 - case 1 : compP.compressionFunction = COMPRESSOR1; break; -#endif - default : compP.compressionFunction = DEFAULTCOMPRESSOR; - } - compP.decompressionFunction = LZ4_decompress_fast; - - // Loop for each file - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - { - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } - - // Alloc - chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*)malloc((size_t )benchedSize); - nbChunks = (int) (benchedSize / chunkSize) + 1; - maxCompressedChunkSize = LZ4_compressBound(chunkSize); - compressedBuffSize = nbChunks * maxCompressedChunkSize; - compressedBuffer = (char*)malloc((size_t )compressedBuffSize); - - - if (!orig_buff || !compressedBuffer) - { - DISPLAY("\nError: not enough memory!\n"); - free(orig_buff); - free(compressedBuffer); - free(chunkP); - fclose(inFile); - return 12; - } - - // Init chunks data - { - int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressedBuffer; - for (i=0; i chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - - // Fill input buffer - DISPLAY("Loading %s... \r", inFileName); - readSize = fread(orig_buff, 1, benchedSize, inFile); - fclose(inFile); - - if (readSize != benchedSize) - { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(orig_buff); - free(compressedBuffer); - free(chunkP); - return 13; - } - - // Calculating input Checksum - crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0); - - - // Bench - { - int loopNb, chunkNb; - size_t cSize=0; - double fastestC = 100000000., fastestD = 100000000.; - double ratio=0.; - U32 crcCheck=0; - - DISPLAY("\r%79s\r", ""); - for (loopNb = 1; loopNb <= nbIterations; loopNb++) - { - int nbLoops; - int milliTime; - - // Compression - DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize); - { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.); - - // Decompression - { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); - - // CRC Checking - crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0); - if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; } - } - - if (crcOrig==crcCheck) - { - if (ratio<100.) - DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); - else - DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); - } - totals += benchedSize; - totalz += cSize; - totalc += fastestC; - totald += fastestD; - } - - free(orig_buff); - free(compressedBuffer); - free(chunkP); - } - - if (nbFiles > 1) - DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.); - - if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); } - - return 0; -} - - - diff --git a/bench.h b/bench.h deleted file mode 100644 index ed801d4..0000000 --- a/bench.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - bench.h - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2013 - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel); - -// Parameters -void BMK_SetBlocksize(int bsize); -void BMK_SetNbIterations(int nbLoops); -void BMK_SetPause(); - - - -#if defined (__cplusplus) -} -#endif diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt deleted file mode 100644 index b922641..0000000 --- a/cmake/CMakeLists.txt +++ /dev/null @@ -1,80 +0,0 @@ -PROJECT(LZ4 C) -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library") -set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 0) -set(CPACK_PACKAGE_VERSION_PATCH r110) -#set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_BINARY_DIR}/COPYING_LGPL) -set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ") -include(CPack) - -cmake_minimum_required (VERSION 2.6) -INCLUDE (CheckTypeSize) -check_type_size("void *" SIZEOF_VOID_P) -IF( ${SIZEOF_VOID_P} STREQUAL "8" ) - set (CMAKE_SYSTEM_PROCESSOR "64bit") - MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P}) -ENDIF() - -option(BUILD_TOOLS "Build the command line tools" ON) -option(BUILD_LIBS "Build the libraries in addition to the tools" OFF) - -if(UNIX AND BUILD_LIBS) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") - add_definitions(-fPIC) - endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") -endif() - -set(SRC_DIR ../) -set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ) -set(LZ4_SRCS ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c) - -if(BUILD_TOOLS AND NOT BUILD_LIBS) - set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB}) -endif() - -if(BUILD_TOOLS) - add_executable(lz4 ${LZ4_SRCS}) - set_target_properties(lz4 PROPERTIES COMPILE_DEFINITIONS DISABLE_LZ4C_LEGACY_OPTIONS) - install(TARGETS lz4 RUNTIME DESTINATION "bin/") - add_executable(lz4c ${LZ4_SRCS}) - install(TARGETS lz4c RUNTIME DESTINATION "bin/") -endif() - -if(BUILD_LIBS) - add_library(liblz4 ${LZ4_SRCS_LIB}) - - set_target_properties(liblz4 PROPERTIES - OUTPUT_NAME lz4 - SOVERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}" - ) - - install(TARGETS liblz4 - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - ) - - install(FILES - ${SRC_DIR}/lz4.h - ${SRC_DIR}/lz4hc.h - DESTINATION include - ) - - if(BUILD_TOOLS) - target_link_libraries(lz4 liblz4) - target_link_libraries(lz4c liblz4) - endif() -endif() - - -#warnings - -ADD_DEFINITIONS("-Wall") -ADD_DEFINITIONS("-W") -ADD_DEFINITIONS("-Wundef") -ADD_DEFINITIONS("-Wcast-align") -ADD_DEFINITIONS("-std=c99") -ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"") -INCLUDE_DIRECTORIES (${SRC_DIR}) - - - diff --git a/cmake/pack/CMakeLists.txt b/cmake/pack/CMakeLists.txt deleted file mode 100644 index 7b93cba..0000000 --- a/cmake/pack/CMakeLists.txt +++ /dev/null @@ -1,82 +0,0 @@ -cmake_minimum_required (VERSION 2.8) -PROJECT(LZ4) - -############################## CPACK -set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../) -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 Packer") -set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 0) -set(CPACK_PACKAGE_VERSION_PATCH r107) -#set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_BINARY_DIR}/COPYING_LGPL) -############################## -FIND_PACKAGE(Subversion) -IF(SUBVERSION_FOUND) - Subversion_WC_INFO(${SRC_DIR} revision) - set(revision_MY_WC_STATUS "LZ4 has revision ${revision_WC_REVISION} at ${revision_WC_LAST_CHANGED_DATE}") - message(STATUS ${revision_MY_WC_STATUS}) - if(NOT ${revision_WC_REVISION}) - else(NOT ${revision_WC_REVISION}) - set(CPACK_PACKAGE_VERSION_PATCH "r${revision_WC_REVISION}") - endif(NOT ${revision_WC_REVISION}) -ELSE(SUBVERSION_FOUND) - message(WARNING "NO Subversion FOUND!!!") -ENDIF(SUBVERSION_FOUND) -set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ") -############################## -INCLUDE (CheckTypeSize) -check_type_size("void *" SIZEOF_VOID_P) -IF( ${SIZEOF_VOID_P} STREQUAL "8" ) - set (CMAKE_SYSTEM_PROCESSOR "64bit") - MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P}) -ENDIF() -############################### warnings - -ADD_DEFINITIONS("-Wall") -ADD_DEFINITIONS("-W") -ADD_DEFINITIONS("-Wundef") -ADD_DEFINITIONS("-Wcast-align") -ADD_DEFINITIONS("-Wno-implicit-function-declaration") -ADD_DEFINITIONS("-O3 -march=native -std=c99") -INCLUDE_DIRECTORIES (${SRC_DIR}) - - - -set(LZ4_SRCS_LIB ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ${SRC_DIR}lz4_format_description.txt) -set(LZ4_SRCS ${LZ4_SRCS_LIB} ${SRC_DIR}xxhash.c ${SRC_DIR}bench.c ${SRC_DIR}lz4cli.c ) -set(FUZZER_SRCS ${SRC_DIR}lz4.c ${SRC_DIR}lz4hc.c ${SRC_DIR}lz4.h ${SRC_DIR}fuzzer.c) - -# EXECUTABLES FOR 32 Bit and 64 versions -if(CMAKE_SYSTEM_PROCESSOR STREQUAL "64bit") - add_executable(lz4c32 ${LZ4_SRCS}) - install(TARGETS lz4c32 RUNTIME DESTINATION "bin/") -SET_TARGET_PROPERTIES(lz4c32 PROPERTIES - COMPILE_FLAGS PROPERTIES COMPILE_FLAGS "-m32 -Os" LINK_FLAGS "-m32") -endif() - -add_executable(lz4c ${LZ4_SRCS}) -install(TARGETS lz4c RUNTIME DESTINATION "bin/") - -add_executable(fuzzer ${FUZZER_SRCS}) -install(TARGETS fuzzer RUNTIME DESTINATION "bin/") - -#target_link_libraries(lz4 ${LZ4_SRCS_LIB}) -####################### CPACK PACKAGING ################### -install(FILES ${SRC_DIR}lz4_format_description.txt DESTINATION "./") - -set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) -set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_LIST_DIR}/release_COPYING.txt") -set(CPACK_PACKAGE_VENDOR "Yann Collet") -set(CPACK_WWW_SITE "http://fastcompression.blogspot.com/p/lz4.html") -set(CPACK_NSIS_URL_INFO_ABOUT "${CPACK_WWW_SITE}") -set(CPACK_NSIS_HELP_LINK "${CPACK_WWW_SITE}") - -set(CPACK_NSIS_DISPLAY_NAME ${CPACK_PACKAGE_NAME}) -set(CPACK_NSIS_COMPRESSOR "/SOLID lzma \r\n SetCompressorDictSize 32") - -set(CPACK_NSIS_MENU_LINKS - "${CPACK_WWW_SITE}" "${CPACK_PACKAGE_NAME} homepage" - "/lz4_format_description.txt " "lz4 format description" - "/" "LZ4 directory " -) - -include (CPack) diff --git a/cmake/pack/release_COPYING.txt b/cmake/pack/release_COPYING.txt deleted file mode 100644 index 32cec0d..0000000 --- a/cmake/pack/release_COPYING.txt +++ /dev/null @@ -1,21 +0,0 @@ - - lz4demo and fuzzer is an open-source demo compression algorithm LZ4 programs - Copyright (C) Yann Collet 2012 - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt new file mode 100644 index 0000000..356f18a --- /dev/null +++ b/cmake_unofficial/CMakeLists.txt @@ -0,0 +1,81 @@ +PROJECT(LZ4 C) +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LZ4 compression library") +set(CPACK_PACKAGE_VERSION_MAJOR 0) +set(CPACK_PACKAGE_VERSION_MINOR 0) +set(CPACK_PACKAGE_VERSION_PATCH r111) +#set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_BINARY_DIR}/COPYING_LGPL) +set(VERSION_STRING " \"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}\" ") +include(CPack) + +cmake_minimum_required (VERSION 2.6) +INCLUDE (CheckTypeSize) +check_type_size("void *" SIZEOF_VOID_P) +IF( ${SIZEOF_VOID_P} STREQUAL "8" ) + set (CMAKE_SYSTEM_PROCESSOR "64bit") + MESSAGE( STATUS "64 bit architecture detected size of void * is " ${SIZEOF_VOID_P}) +ENDIF() + +option(BUILD_TOOLS "Build the command line tools" ON) +option(BUILD_LIBS "Build the libraries in addition to the tools" OFF) + +if(UNIX AND BUILD_LIBS) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") + add_definitions(-fPIC) + endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64") +endif() + +set(LZ4_DIR ../) +set(PRG_DIR ../programs/) +set(LZ4_SRCS_LIB ${LZ4_DIR}lz4.c ${LZ4_DIR}lz4hc.c ${LZ4_DIR}lz4.h ${LZ4_DIR}lz4hc.h) +set(LZ4_SRCS ${PRG_DIR}xxhash.c ${PRG_DIR}bench.c ${PRG_DIR}lz4cli.c) + +if(BUILD_TOOLS AND NOT BUILD_LIBS) + set(LZ4_SRCS ${LZ4_SRCS} ${LZ4_SRCS_LIB}) +endif() + +if(BUILD_TOOLS) + add_executable(lz4 ${LZ4_SRCS}) + set_target_properties(lz4 PROPERTIES COMPILE_DEFINITIONS DISABLE_LZ4C_LEGACY_OPTIONS) + install(TARGETS lz4 RUNTIME DESTINATION "bin/") + add_executable(lz4c ${LZ4_SRCS}) + install(TARGETS lz4c RUNTIME DESTINATION "bin/") +endif() + +if(BUILD_LIBS) + add_library(liblz4 ${LZ4_SRCS_LIB}) + + set_target_properties(liblz4 PROPERTIES + OUTPUT_NAME lz4 + SOVERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}" + ) + + install(TARGETS liblz4 + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + ) + + install(FILES + ${LZ4_DIR}/lz4.h + ${LZ4_DIR}/lz4hc.h + DESTINATION include + ) + + if(BUILD_TOOLS) + target_link_libraries(lz4 liblz4) + target_link_libraries(lz4c liblz4) + endif() +endif() + + +#warnings + +ADD_DEFINITIONS("-Wall") +ADD_DEFINITIONS("-W") +ADD_DEFINITIONS("-Wundef") +ADD_DEFINITIONS("-Wcast-align") +ADD_DEFINITIONS("-std=c99") +ADD_DEFINITIONS("-DLZ4_VERSION=\"${CPACK_PACKAGE_VERSION_PATCH}\"") +INCLUDE_DIRECTORIES (${LZ4_DIR}) + + + diff --git a/fullbench.c b/fullbench.c deleted file mode 100644 index c465c88..0000000 --- a/fullbench.c +++ /dev/null @@ -1,742 +0,0 @@ -/* - bench.c - Demo program to benchmark open-source compression algorithm - Copyright (C) Yann Collet 2012-2013 - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ - -//************************************** -// Compiler Options -//************************************** -// Disable some Visual warning messages -#define _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_DEPRECATE // VS2005 - -// Unix Large Files support (>4GB) -#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions -# define _LARGEFILE_SOURCE -# define _FILE_OFFSET_BITS 64 -#elif ! defined(__LP64__) // No point defining Large file for 64 bit -# define _LARGEFILE64_SOURCE -#endif - -// S_ISREG & gettimeofday() are not supported by MSVC -#if defined(_MSC_VER) || defined(_WIN32) -# define BMK_LEGACY_TIMER 1 -#endif - - -//************************************** -// Includes -//************************************** -#include // malloc -#include // fprintf, fopen, ftello64 -#include // stat64 -#include // stat64 - -// Use ftime() if gettimeofday() is not available on your target -#if defined(BMK_LEGACY_TIMER) -# include // timeb, ftime -#else -# include // gettimeofday -#endif - -#include "lz4.h" -#define COMPRESSOR0 LZ4_compress -#include "lz4hc.h" -#define COMPRESSOR1 LZ4_compressHC -#define DEFAULTCOMPRESSOR COMPRESSOR0 - -#include "xxhash.h" - - -//************************************** -// Compiler Options -//************************************** -// S_ISREG & gettimeofday() are not supported by MSVC -#if !defined(S_ISREG) -# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) -#endif - -// GCC does not support _rotl outside of Windows -#if !defined(_WIN32) -# define _rotl(x,r) ((x << r) | (x >> (32 - r))) -#endif - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - - -//**************************** -// Constants -//**************************** -#define PROGRAM_DESCRIPTION "LZ4 speed analyzer" -#ifndef LZ4_VERSION -# define LZ4_VERSION "" -#endif -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__ - -#define NBLOOPS 6 -#define TIMELOOP 2500 - -#define KNUTH 2654435761U -#define MAX_MEM (1984<<20) -#define DEFAULT_CHUNKSIZE (4<<20) - -#define ALL_COMPRESSORS -1 -#define ALL_DECOMPRESSORS -1 - - -//************************************** -// Local structures -//************************************** -struct chunkParameters -{ - U32 id; - char* origBuffer; - char* compressedBuffer; - int origSize; - int compressedSize; -}; - - -//************************************** -// MACRO -//************************************** -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) - - - -//************************************** -// Benchmark Parameters -//************************************** -static int chunkSize = DEFAULT_CHUNKSIZE; -static int nbIterations = NBLOOPS; -static int BMK_pause = 0; -static int compressionTest = 1; -static int decompressionTest = 1; -static int compressionAlgo = ALL_COMPRESSORS; -static int decompressionAlgo = ALL_DECOMPRESSORS; - -void BMK_SetBlocksize(int bsize) -{ - chunkSize = bsize; - DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10); -} - -void BMK_SetNbIterations(int nbLoops) -{ - nbIterations = nbLoops; - DISPLAY("- %i iterations -\n", nbIterations); -} - -void BMK_SetPause() -{ - BMK_pause = 1; -} - -//********************************************************* -// Private functions -//********************************************************* - -#if defined(BMK_LEGACY_TIMER) - -static int BMK_GetMilliStart() -{ - // Based on Legacy ftime() - // Rolls over every ~ 12.1 days (0x100000/24/60/60) - // Use GetMilliSpan to correct for rollover - struct timeb tb; - int nCount; - ftime( &tb ); - nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); - return nCount; -} - -#else - -static int BMK_GetMilliStart() -{ - // Based on newer gettimeofday() - // Use GetMilliSpan to correct for rollover - struct timeval tv; - int nCount; - gettimeofday(&tv, NULL); - nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); - return nCount; -} - -#endif - - -static int BMK_GetMilliSpan( int nTimeStart ) -{ - int nSpan = BMK_GetMilliStart() - nTimeStart; - if ( nSpan < 0 ) - nSpan += 0x100000 * 1000; - return nSpan; -} - - -static size_t BMK_findMaxMem(U64 requiredMem) -{ - size_t step = (64U<<20); // 64 MB - BYTE* testmem=NULL; - - requiredMem = (((requiredMem >> 25) + 1) << 26); - if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; - - requiredMem += 2*step; - while (!testmem) - { - requiredMem -= step; - testmem = (BYTE*) malloc ((size_t)requiredMem); - } - - free (testmem); - return (size_t) (requiredMem - step); -} - - -static U64 BMK_GetFileSize(char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... - return (U64)statbuf.st_size; -} - - -//********************************************************* -// Public function -//********************************************************* - -static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); -} - -static void* stateLZ4; -static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize) -{ - return LZ4_compress_withState(stateLZ4, in, out, inSize); -} - -static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)); -} - -static void* ctx; -static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize) -{ - return LZ4_compress_continue(ctx, in, out, inSize); -} - -static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); -} - -static void* stateLZ4HC; -static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) -{ - return LZ4_compress_withState(stateLZ4HC, in, out, inSize); -} - -static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) -{ - return LZ4_compress_limitedOutput_withState(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); -} - -static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); -} - -static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_continue(ctx, in, out, inSize); -} - -static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) -{ - return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); -} - -static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_fast(in, out, outSize); - return outSize; -} - -static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize) -{ - (void)inSize; - LZ4_decompress_fast_withPrefix64k(in, out, outSize); - return outSize; -} - -static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) -{ - return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); -} - -int fullSpeedBench(char** fileNamesTable, int nbFiles) -{ - int fileIdx=0; - char* orig_buff; -# define NB_COMPRESSION_ALGORITHMS 12 -# define MINCOMPRESSIONCHAR '0' -# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS) - static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", - "LZ4_compress_withState", "LZ4_compress_limitedOutput_withState", - "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue", - "LZ4_compressHC", "LZ4_compressHC_limitedOutput", - "LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC", - "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" }; - double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0}; - double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0}; -# define NB_DECOMPRESSION_ALGORITHMS 5 -# define MINDECOMPRESSIONCHAR '0' -# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS) - static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" }; - double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0}; - - U64 totals = 0; - - - // Loop for each file - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - { - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } - - // Alloc - chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters)); - orig_buff = (char*) malloc((size_t)benchedSize); - nbChunks = (int) (benchedSize / chunkSize) + 1; - maxCompressedChunkSize = LZ4_compressBound(chunkSize); - compressedBuffSize = nbChunks * maxCompressedChunkSize; - compressed_buff = (char*)malloc((size_t)compressedBuffSize); - - - if(!orig_buff || !compressed_buff) - { - DISPLAY("\nError: not enough memory!\n"); - free(orig_buff); - free(compressed_buff); - free(chunkP); - fclose(inFile); - return 12; - } - - // Init chunks data - { - int i; - size_t remaining = benchedSize; - char* in = orig_buff; - char* out = compressed_buff; - for (i=0; i chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } - chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; - chunkP[i].compressedSize = 0; - } - } - - // Fill input buffer - DISPLAY("Loading %s... \r", inFileName); - readSize = fread(orig_buff, 1, benchedSize, inFile); - fclose(inFile); - - if(readSize != benchedSize) - { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(orig_buff); - free(compressed_buff); - free(chunkP); - return 13; - } - - // Calculating input Checksum - crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0); - - - // Bench - { - int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; - size_t cSize=0; - double ratio=0.; - - DISPLAY("\r%79s\r", ""); - DISPLAY(" %s : \n", inFileName); - - // Compression Algorithms - for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) - { - char* cName = compressionNames[cAlgNb]; - int (*compressionFunction)(const char*, char*, int); - void* (*initFunction)(const char*) = NULL; - double bestTime = 100000000.; - - if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; - - switch(cAlgNb) - { - case 0 : compressionFunction = LZ4_compress; break; - case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break; - case 2 : compressionFunction = local_LZ4_compress_withState; break; - case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break; - case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break; - case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break; - case 6 : compressionFunction = LZ4_compressHC; break; - case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break; - case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break; - case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break; - case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break; - case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break; - default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; - } - - for (loopNb = 1; loopNb <= nbIterations; loopNb++) - { - double averageTime; - int milliTime; - - DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize); - { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); - } - - if (ratio<100.) - DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); - else - DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); - - totalCTime[cAlgNb] += bestTime; - totalCSize[cAlgNb] += cSize; - } - - // Prepare layout for decompression - for (chunkNb=0; chunkNb\r", loopNb, dName, (int)benchedSize); - - nb_loops = 0; - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliSpan(milliTime) < TIMELOOP) - { - for (chunkNb=0; chunkNb %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); - - // CRC Checking - crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); - if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } - } - - DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); - - totalDTime[dAlgNb] += bestTime; - } - - totals += benchedSize; - } - - free(orig_buff); - free(compressed_buff); - free(chunkP); - } - - if (nbFiles > 1) - { - int AlgNb; - - DISPLAY(" ** TOTAL ** : \n"); - for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++) - { - char* cName = compressionNames[AlgNb]; - if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue; - DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.); - } - for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++) - { - char* dName = decompressionNames[AlgNb]; - if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue; - DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.); - } - } - - if (BMK_pause) { printf("press enter...\n"); getchar(); } - - return 0; -} - - -int usage(char* exename) -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); - DISPLAY( "Arguments :\n"); - DISPLAY( " -c : compression tests only\n"); - DISPLAY( " -d : decompression tests only\n"); - DISPLAY( " -H/-h : Help (this text + advanced options)\n"); - return 0; -} - -int usage_advanced() -{ - DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR); - DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR); - DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); - DISPLAY( " -B# : Block size [4-7](default : 7)\n"); - //DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); - return 0; -} - -int badusage(char* exename) -{ - DISPLAY("Wrong parameters\n"); - usage(exename); - return 0; -} - -int main(int argc, char** argv) -{ - int i, - filenamesStart=2; - char* exename=argv[0]; - char* input_filename=0; - - // Welcome message - DISPLAY( WELCOME_MESSAGE); - - if (argc<2) { badusage(exename); return 1; } - - for(i=1; i= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR)) - compressionAlgo = argument[1] - '0', argument++; - break; - - // Select decompression algorithm only - case 'd': - compressionTest = 0; - if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR)) - decompressionAlgo = argument[1] - '0', argument++; - break; - - // Display help on usage - case 'h' : - case 'H': usage(exename); usage_advanced(); return 0; - - // Modify Block Properties - case 'B': - while (argument[1]!=0) - switch(argument[1]) - { - case '4': - case '5': - case '6': - case '7': - { - int B = argument[1] - '0'; - int S = 1 << (8 + 2*B); - BMK_SetBlocksize(S); - argument++; - break; - } - case 'D': argument++; break; - default : goto _exit_blockProperties; - } -_exit_blockProperties: - break; - - // Modify Nb Iterations - case 'i': - if ((argument[1] >='1') && (argument[1] <='9')) - { - int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); - argument++; - } - break; - - // Pause at the end (hidden option) - case 'p': BMK_SetPause(); break; - - // Unrecognised command - default : badusage(exename); return 1; - } - } - continue; - } - - // first provided filename is input - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - - } - - // No input filename ==> Error - if(!input_filename) { badusage(exename); return 1; } - - return fullSpeedBench(argv+filenamesStart, argc-filenamesStart); - -} - diff --git a/fuzzer.c b/fuzzer.c deleted file mode 100644 index 4513ebe..0000000 --- a/fuzzer.c +++ /dev/null @@ -1,303 +0,0 @@ -/* - fuzzer.c - Fuzzer test tool for LZ4 - Copyright (C) Yann Collet - Andrew Mahone 2012-2013 - Code started by Andrew Mahone, modified by Yann Collet - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ - -//************************************** -// Remove Visual warning messages -//************************************** -#define _CRT_SECURE_NO_WARNINGS // fgets - - -//************************************** -// Includes -//************************************** -#include -#include // fgets, sscanf -#include // timeb -#include "lz4.h" -#include "lz4hc.h" - - -//************************************** -// Constants -//************************************** -#ifndef LZ4_VERSION -# define LZ4_VERSION "" -#endif - -#define NB_ATTEMPTS (1<<17) -#define LEN ((1<<15)) -#define SEQ_POW 2 -#define NUM_SEQ (1 << SEQ_POW) -#define SEQ_MSK ((NUM_SEQ) - 1) -#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0) -#define NEW_SEQ(x) ((((x) >> 10) %10) == 0) -#define PAGE_SIZE 4096 -#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) -#define PRIME1 2654435761U -#define PRIME2 2246822519U -#define PRIME3 3266489917U - - -//********************************************************* -// Functions -//********************************************************* -static int FUZ_GetMilliStart() -{ - struct timeb tb; - int nCount; - ftime( &tb ); - nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); - return nCount; -} - - -static int FUZ_GetMilliSpan( int nTimeStart ) -{ - int nSpan = FUZ_GetMilliStart() - nTimeStart; - if ( nSpan < 0 ) - nSpan += 0x100000 * 1000; - return nSpan; -} - - -unsigned int FUZ_rand(unsigned int* src) -{ - *src = ((*src) * PRIME1) + PRIME2; - return *src; -} - - -int test_canary(unsigned char *buf) -{ - int i; - for (i = 0; i < 2048; i++) - if (buf[i] != buf[i + 2048]) - return 0; - return 1; -} - - -int FUZ_SecurityTest() -{ - char* output; - char* input; - int i, r; - - printf("Overflow test (issue 52)...\n"); - input = (char*) malloc (20<<20); - output = (char*) malloc (20<<20); - input[0] = 0x0F; - input[1] = 0x00; - input[2] = 0x00; - for(i = 3; i < 16840000; i++) - input[i] = 0xff; - r = LZ4_decompress_fast(input, output, 20<<20); - - free(input); - free(output); - printf(" Passed (return = %i < 0)\n",r); - return 0; -} - - -//int main(int argc, char *argv[]) { -int main() { - unsigned long long bytes = 0; - unsigned long long cbytes = 0; - unsigned long long hcbytes = 0; - unsigned char buf[LEN]; - unsigned char testOut[LEN+1]; -# define FUZ_max LZ4_COMPRESSBOUND(LEN) -# define FUZ_avail ROUND_PAGE(FUZ_max) - const int off_full = FUZ_avail - FUZ_max; - unsigned char cbuf[FUZ_avail + PAGE_SIZE]; - unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); - int i, j, k, ret, len, lenHC, attemptNb; - char userInput[30] = {0}; -# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; } -# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb); - void* stateLZ4 = malloc(LZ4_sizeofState()); - void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); - - printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION); - printf("Select an Initialisation number (default : random) : "); - fflush(stdout); - if ( fgets(userInput, sizeof userInput, stdin) ) - { - if ( sscanf(userInput, "%d", &seed) == 1 ) {} - else seed = FUZ_GetMilliSpan(timestamp); - } - printf("Seed = %u\n", seed); - randState = seed; - - //FUZ_SecurityTest(); - - for (i = 0; i < 2048; i++) - cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16; - - for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++) - { - int testNb = 0; - - printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS); - - for (j = 0; j < NUM_SEQ; j++) { - seeds[j] = FUZ_rand(&randState) << 8; - seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535; - } - for (j = 0; j < LEN; j++) { - k = FUZ_rand(&randState); - if (j == 0 || NEW_SEQ(k)) - cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK]; - if (MOD_SEQ(k)) { - k = (FUZ_rand(&randState) >> 16) & SEQ_MSK; - seeds[k] = FUZ_rand(&randState) << 8; - seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535; - } - buf[j] = FUZ_rand(&cur_seq) >> 16; - } - - // Test compression HC - FUZ_DISPLAYTEST; // 1 - ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); - lenHC = ret; - - // Test compression HC using external state - FUZ_DISPLAYTEST; // 1 - ret = LZ4_compressHC_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[off_full], LEN); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); - - // Test compression using external state - FUZ_DISPLAYTEST; // 2 - ret = LZ4_compress_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN); - FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); - - // Test compression - FUZ_DISPLAYTEST; // 2 - ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); - len = ret; - - // Test decoding with output size being exactly what's necessary => must work - FUZ_DISPLAYTEST; // 3 - ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); - - // Test decoding with one byte missing => must fail - FUZ_DISPLAYTEST; // 4 - ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); - - // Test decoding with one byte too much => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); - - // Test decoding with enough output size => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); - - // Test decoding with output size being exactly what's necessary => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); - - // Test decoding with output size being one byte too short => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); - - // Test decoding with input size being one byte too short => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short"); - - // Test decoding with input size being one byte too large => must fail - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); - FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); - //if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } - - // Test partial decoding with target output size being max/2 => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - // Test partial decoding with target output size being just below max => must work - FUZ_DISPLAYTEST; - ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN); - FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); - - // Test compression with output size being exactly what's necessary (should work) - FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); - FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); - - // Test compression with output size being exactly what's necessary and external state (should work) - FUZ_DISPLAYTEST; // 2 - ret = LZ4_compress_limitedOutput_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN, len); - FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); - FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); - - // Test HC compression with output size being exactly what's necessary (should work) - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); - - // Test HC compression with output size being exactly what's necessary (should work) - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC); - FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); - - // Test compression with just one missing byte into output buffer => must fail - FUZ_DISPLAYTEST; - ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); - FUZ_CHECKTEST(ret, "compression overran output buffer"); - FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); - - // Test HC compression with just one missing byte into output buffer => must fail - FUZ_DISPLAYTEST; - ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1); - FUZ_CHECKTEST(ret, "HC compression overran output buffer"); - - bytes += LEN; - cbytes += len; - hcbytes += lenHC; - FUZ_rand(&randState); - } - - printf("all tests completed successfully \n"); - printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); - printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); - getchar(); - return 0; - -_output_error: - getchar(); - return 1; -} diff --git a/lz4.1 b/lz4.1 deleted file mode 100644 index 69c58c3..0000000 --- a/lz4.1 +++ /dev/null @@ -1,87 +0,0 @@ -\" -\" lz4.1: This is a manual page for 'lz4' program. This file is part of the -\" lz4 project. -\" - -\" No hyphenation -.hy 0 -.nr HY 0 - -.TH lz4 man -.SH NAME -\fBlz4\fR - Extremely fast compression algorithm - -.SH SYNOPSIS -.TP 5 -\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] - -.SH DESCRIPTION -.PP -\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on -the \fBLZ77\fR family of compression scheme. At the compression speed of 400 -MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features -an extremely fast decoder, with speed in multiple GB/s per core, typically -reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports -following options - -.SH OPTIONS -.TP -.B \-1 - fast compression (default) -.TP -.B \-9 - high compression -.TP -.B \-d - decompression -.TP -.B \-f - overwrite output without prompting -.TP -.B \-h/\-H - display help/long help and exit -.TP -.B \-V - display Version number and exit -.TP -.B \-v - verbose mode -.TP -.B \-q - suppress warnings; specify twice to suppress errors too -.TP -.B \-c - force write to standard output, even if it is the console -.TP -.B \-t - test compressed file integrity -.TP -.B \-z - force compression -.TP -.B \-l - use Legacy format (useful for Linux Kernel compression) -.TP -.B \-B# - block size [4-7](default : 7) -.TP -.B \-BD - block dependency (improve compression ratio) -.TP -.B \-BX - enable block checksum (default:disabled) -.TP -.B \-Sx - disable stream checksum (default:enabled) -.TP -.B \-b - benchmark file(s) -.TP -.B \-i# - iteration loops [1-9](default : 3), benchmark mode only - -.SH BUGS -Report bugs at:- https://code.google.com/p/lz4/ - -.SH AUTHOR -Yann Collet \ No newline at end of file diff --git a/lz4_format_description.txt b/lz4_format_description.txt index 888c57b..2c424c5 100644 --- a/lz4_format_description.txt +++ b/lz4_format_description.txt @@ -1,120 +1,120 @@ -LZ4 Format Description -Last revised: 2012-02-27 -Author : Y. Collet - - - -This small specification intents to provide enough information -to anyone willing to produce LZ4-compatible compressed data blocks -using any programming language. - -LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding. -The most important design principle behind LZ4 is simplicity. -It helps to create an easy to read and maintain source code. -It also helps later on for optimisations, compactness, and speed. -There is no entropy encoder backend nor framing layer. -The latter is assumed to be handled by other parts of the system. - -This document only describes the format, -not how the LZ4 compressor nor decompressor actually work. -The correctness of the decompressor should not depend -on implementation details of the compressor, and vice versa. - - - --- Compressed block format -- - -An LZ4 compressed block is composed of sequences. -Schematically, a sequence is a suite of literals, followed by a match copy. - -Each sequence starts with a token. -The token is a one byte value, separated into two 4-bits fields. -Therefore each field ranges from 0 to 15. - - -The first field uses the 4 high-bits of the token. -It provides the length of literals to follow. -(Note : a literal is a not-compressed byte). -If the field value is 0, then there is no literal. -If it is 15, then we need to add some more bytes to indicate the full length. -Each additionnal byte then represent a value from 0 to 255, -which is added to the previous value to produce a total length. -When the byte value is 255, another byte is output. -There can be any number of bytes following the token. There is no "size limit". -(Sidenote this is why a not-compressible input block is expanded by 0.4%). - -Example 1 : A length of 48 will be represented as : -- 15 : value for the 4-bits High field -- 33 : (=48-15) remaining length to reach 48 - -Example 2 : A length of 280 will be represented as : -- 15 : value for the 4-bits High field -- 255 : following byte is maxed, since 280-15 >= 255 -- 10 : (=280 - 15 - 255) ) remaining length to reach 280 - -Example 3 : A length of 15 will be represented as : -- 15 : value for the 4-bits High field -- 0 : (=15-15) yes, the zero must be output - -Following the token and optional length bytes, are the literals themselves. -They are exactly as numerous as previously decoded (length of literals). -It's possible that there are zero literal. - - -Following the literals is the match copy operation. - -It starts by the offset. -This is a 2 bytes value, in little endian format. - -The offset represents the position of the match to be copied from. -1 means "current position - 1 byte". -The maximum offset value is 65535, 65536 cannot be coded. -Note that 0 is an invalid value, not used. - -Then we need to extract the match length. -For this, we use the second token field, the low 4-bits. -Value, obviously, ranges from 0 to 15. -However here, 0 means that the copy operation will be minimal. -The minimum length of a match, called minmatch, is 4. -As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes. -Similar to literal length, on reaching the highest possible value (15), -we output additional bytes, one at a time, with values ranging from 0 to 255. -They are added to total to provide the final match length. -A 255 value means there is another byte to read and add. -There is no limit to the number of optional bytes that can be output this way. -(This points towards a maximum achievable compression ratio of ~250). - -With the offset and the matchlength, -the decoder can now proceed to copy the data from the already decoded buffer. -On decoding the matchlength, we reach the end of the compressed sequence, -and therefore start another one. - - --- Parsing restrictions -- - -There are specific parsing rules to respect in order to remain compatible -with assumptions made by the decoder : -1) The last 5 bytes are always literals -2) The last match must start at least 12 bytes before end of block -Consequently, a block with less than 13 bytes cannot be compressed. -These rules are in place to ensure that the decoder -will never read beyond the input buffer, nor write beyond the output buffer. - -Note that the last sequence is also incomplete, -and stops right after literals. - - --- Additional notes -- - -There is no assumption nor limits to the way the compressor -searches and selects matches within the source data block. -It could be a fast scan, a multi-probe, a full search using BST, -standard hash chains or MMC, well whatever. - -Advanced parsing strategies can also be implemented, such as lazy match, -or full optimal parsing. - -All these trade-off offer distinctive speed/memory/compression advantages. -Whatever the method used by the compressor, its result will be decodable -by any LZ4 decoder if it follows the format specification described above. - +LZ4 Format Description +Last revised: 2012-02-27 +Author : Y. Collet + + + +This small specification intents to provide enough information +to anyone willing to produce LZ4-compatible compressed data blocks +using any programming language. + +LZ4 is an LZ77-type compressor with a fixed, byte-oriented encoding. +The most important design principle behind LZ4 is simplicity. +It helps to create an easy to read and maintain source code. +It also helps later on for optimisations, compactness, and speed. +There is no entropy encoder backend nor framing layer. +The latter is assumed to be handled by other parts of the system. + +This document only describes the format, +not how the LZ4 compressor nor decompressor actually work. +The correctness of the decompressor should not depend +on implementation details of the compressor, and vice versa. + + + +-- Compressed block format -- + +An LZ4 compressed block is composed of sequences. +Schematically, a sequence is a suite of literals, followed by a match copy. + +Each sequence starts with a token. +The token is a one byte value, separated into two 4-bits fields. +Therefore each field ranges from 0 to 15. + + +The first field uses the 4 high-bits of the token. +It provides the length of literals to follow. +(Note : a literal is a not-compressed byte). +If the field value is 0, then there is no literal. +If it is 15, then we need to add some more bytes to indicate the full length. +Each additionnal byte then represent a value from 0 to 255, +which is added to the previous value to produce a total length. +When the byte value is 255, another byte is output. +There can be any number of bytes following the token. There is no "size limit". +(Sidenote this is why a not-compressible input block is expanded by 0.4%). + +Example 1 : A length of 48 will be represented as : +- 15 : value for the 4-bits High field +- 33 : (=48-15) remaining length to reach 48 + +Example 2 : A length of 280 will be represented as : +- 15 : value for the 4-bits High field +- 255 : following byte is maxed, since 280-15 >= 255 +- 10 : (=280 - 15 - 255) ) remaining length to reach 280 + +Example 3 : A length of 15 will be represented as : +- 15 : value for the 4-bits High field +- 0 : (=15-15) yes, the zero must be output + +Following the token and optional length bytes, are the literals themselves. +They are exactly as numerous as previously decoded (length of literals). +It's possible that there are zero literal. + + +Following the literals is the match copy operation. + +It starts by the offset. +This is a 2 bytes value, in little endian format. + +The offset represents the position of the match to be copied from. +1 means "current position - 1 byte". +The maximum offset value is 65535, 65536 cannot be coded. +Note that 0 is an invalid value, not used. + +Then we need to extract the match length. +For this, we use the second token field, the low 4-bits. +Value, obviously, ranges from 0 to 15. +However here, 0 means that the copy operation will be minimal. +The minimum length of a match, called minmatch, is 4. +As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes. +Similar to literal length, on reaching the highest possible value (15), +we output additional bytes, one at a time, with values ranging from 0 to 255. +They are added to total to provide the final match length. +A 255 value means there is another byte to read and add. +There is no limit to the number of optional bytes that can be output this way. +(This points towards a maximum achievable compression ratio of ~250). + +With the offset and the matchlength, +the decoder can now proceed to copy the data from the already decoded buffer. +On decoding the matchlength, we reach the end of the compressed sequence, +and therefore start another one. + + +-- Parsing restrictions -- + +There are specific parsing rules to respect in order to remain compatible +with assumptions made by the decoder : +1) The last 5 bytes are always literals +2) The last match must start at least 12 bytes before end of block +Consequently, a block with less than 13 bytes cannot be compressed. +These rules are in place to ensure that the decoder +will never read beyond the input buffer, nor write beyond the output buffer. + +Note that the last sequence is also incomplete, +and stops right after literals. + + +-- Additional notes -- + +There is no assumption nor limits to the way the compressor +searches and selects matches within the source data block. +It could be a fast scan, a multi-probe, a full search using BST, +standard hash chains or MMC, well whatever. + +Advanced parsing strategies can also be implemented, such as lazy match, +or full optimal parsing. + +All these trade-off offer distinctive speed/memory/compression advantages. +Whatever the method used by the compressor, its result will be decodable +by any LZ4 decoder if it follows the format specification described above. + diff --git a/lz4cli.c b/lz4cli.c deleted file mode 100644 index f69c8a0..0000000 --- a/lz4cli.c +++ /dev/null @@ -1,1257 +0,0 @@ -/* - LZ4cli.c - LZ4 Command Line Interface - Copyright (C) Yann Collet 2011-2013 - GPL v2 License - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ -/* - Note : this is stand-alone program. - It is not part of LZ4 compression library, it is a user program of the LZ4 library. - The license of LZ4 library is BSD. - The license of xxHash library is BSD. - The license of this compression CLI program is GPLv2. -*/ - -//************************************** -// Tuning parameters -//************************************** -// DISABLE_LZ4C_LEGACY_OPTIONS : -// Control the availability of -c0, -c1 and -hc legacy arguments -// Default : Legacy options are enabled -// #define DISABLE_LZ4C_LEGACY_OPTIONS - - -//************************************** -// Compiler Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# define _CRT_SECURE_NO_WARNINGS -# define _CRT_SECURE_NO_DEPRECATE // VS2005 -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix -#define _POSIX_SOURCE 1 // for fileno() within on unix - - -//**************************** -// Includes -//**************************** -#include // fprintf, fopen, fread, _fileno, stdin, stdout -#include // malloc -#include // strcmp, strlen -#include // clock -#include "lz4.h" -#include "lz4hc.h" -#include "xxhash.h" -#include "bench.h" - - -//**************************** -// OS-specific Includes -//**************************** -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include // _O_BINARY -# include // _setmode, _isatty -# ifdef __MINGW32__ - int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into -# endif -# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#else -# include // isatty -# define SET_BINARY_MODE(file) -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#endif - - -//************************************** -// Compiler-specific functions -//************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -#if defined(_MSC_VER) // Visual Studio -# define swap32 _byteswap_ulong -#elif GCC_VERSION >= 403 -# define swap32 __builtin_bswap32 -#else - static inline unsigned int swap32(unsigned int x) - { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); - } -#endif - - -//**************************** -// Constants -//**************************** -#define COMPRESSOR_NAME "LZ4 Compression CLI" -#ifndef LZ4_VERSION -# define LZ4_VERSION "v1.1.0" -#endif -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__ -#define LZ4_EXTENSION ".lz4" - -#define KB *(1U<<10) -#define MB *(1U<<20) -#define GB *(1U<<30) - -#define _1BIT 0x01 -#define _2BITS 0x03 -#define _3BITS 0x07 -#define _4BITS 0x0F -#define _8BITS 0xFF - -#define MAGICNUMBER_SIZE 4 -#define LZ4S_MAGICNUMBER 0x184D2204 -#define LZ4S_SKIPPABLE0 0x184D2A50 -#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0 -#define LEGACY_MAGICNUMBER 0x184C2102 - -#define CACHELINE 64 -#define LEGACY_BLOCKSIZE (8 MB) -#define MIN_STREAM_BUFSIZE (1 MB + 64 KB) -#define LZ4S_BLOCKSIZEID_DEFAULT 7 -#define LZ4S_CHECKSUM_SEED 0 -#define LZ4S_EOS 0 -#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1) - - -//************************************** -// Architecture Macros -//************************************** -static const int one = 1; -#define CPU_LITTLE_ENDIAN (*(char*)(&one)) -#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) -#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i)) - - -//************************************** -// Macros -//************************************** -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } - - -//************************************** -// Special input/output -//************************************** -#define NULL_OUTPUT "null" -char stdinmark[] = "stdin"; -char stdoutmark[] = "stdout"; -#ifdef _WIN32 -char nulmark[] = "nul"; -#else -char nulmark[] = "/dev/null"; -#endif - - -//************************************** -// Local Parameters -//************************************** -static char* programName; -static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information -static int overwrite = 0; -static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; -static int blockChecksum = 0; -static int streamChecksum = 1; -static int blockIndependence = 1; - - -//************************************** -// Exceptions -//************************************** -#define DEBUG 0 -#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); -#define EXM_THROW(error, ...) \ -{ \ - DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ - exit(error); \ -} - - -//************************************** -// Version modifiers -//************************************** -#define EXTENDED_ARGUMENTS -#define EXTENDED_HELP -#define EXTENDED_FORMAT -#define DEFAULT_COMPRESSOR compress_file -#define DEFAULT_DECOMPRESSOR decodeLZ4S - - -//**************************** -// Functions -//**************************** -int usage() -{ - DISPLAY( "Usage :\n"); - DISPLAY( " %s [arg] [input] [output]\n", programName); - DISPLAY( "\n"); - DISPLAY( "input : a filename\n"); - DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); - DISPLAY( "Arguments :\n"); - DISPLAY( " -1 : Fast compression (default) \n"); - DISPLAY( " -9 : High compression \n"); - DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); - DISPLAY( " -z : force compression\n"); - DISPLAY( " -f : overwrite output without prompting \n"); - DISPLAY( " -h/-H : display help/long help and exit\n"); - return 0; -} - -int usage_advanced() -{ - DISPLAY(WELCOME_MESSAGE); - usage(); - DISPLAY( "\n"); - DISPLAY( "Advanced arguments :\n"); - DISPLAY( " -V : display Version number and exit\n"); - DISPLAY( " -v : verbose mode\n"); - DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); - DISPLAY( " -c : force write to standard output, even if it is the console\n"); - DISPLAY( " -t : test compressed file integrity\n"); - DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); - DISPLAY( " -B# : Block size [4-7](default : 7)\n"); - DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); - DISPLAY( " -BX : enable block checksum (default:disabled)\n"); - DISPLAY( " -Sx : disable stream checksum (default:enabled)\n"); - DISPLAY( "Benchmark arguments :\n"); - DISPLAY( " -b : benchmark file(s)\n"); - DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); -#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "Legacy arguments :\n"); - DISPLAY( " -c0 : fast compression\n"); - DISPLAY( " -c1 : high compression\n"); - DISPLAY( " -hc : high compression\n"); - DISPLAY( " -y : overwrite output without prompting \n"); - DISPLAY( " -s : suppress warnings \n"); -#endif // DISABLE_LZ4C_LEGACY_OPTIONS - EXTENDED_HELP; - return 0; -} - -int usage_longhelp() -{ - DISPLAY( "\n"); - DISPLAY( "Which values can get [output] ? \n"); - DISPLAY( "[output] : a filename\n"); - DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark); - DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT); - DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n"); - DISPLAY( " - if stdout is not the console, then [output] = stdout \n"); - DISPLAY( " - if stdout is console : \n"); - DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION); - DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION); - DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION); - DISPLAY( "\n"); - DISPLAY( "Compression levels : \n"); - DISPLAY( "There are technically 2 accessible compression levels.\n"); - DISPLAY( "-0 ... -2 => Fast compression\n"); - DISPLAY( "-3 ... -9 => High compression\n"); - DISPLAY( "\n"); - DISPLAY( "stdin, stdout and the console : \n"); - DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); - DISPLAY( "%s will refuse to read from console, or write to console \n", programName); - DISPLAY( "except if '-c' command is specified, to force output to console \n"); - DISPLAY( "\n"); - DISPLAY( "Simple example :\n"); - DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n"); - DISPLAY( " %s filename\n", programName); - DISPLAY( "\n"); - DISPLAY( "Arguments can be appended together, or provided independently. For example :\n"); - DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); - DISPLAY( " %s -f9 filename \n", programName); - DISPLAY( " is equivalent to :\n"); - DISPLAY( " %s -f -9 filename \n", programName); - DISPLAY( "\n"); - DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName); - DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); - DISPLAY( " generator | %s | consumer \n", programName); -#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) - DISPLAY( "\n"); - DISPLAY( "Warning :\n"); - DISPLAY( "Legacy arguments take precedence. Therefore : \n"); - DISPLAY( " %s -hc filename\n", programName); - DISPLAY( "means 'compress filename in high compression mode'\n"); - DISPLAY( "It is not equivalent to :\n"); - DISPLAY( " %s -h -c filename\n", programName); - DISPLAY( "which would display help text and exit\n"); -#endif // DISABLE_LZ4C_LEGACY_OPTIONS - return 0; -} - -int badusage() -{ - DISPLAYLEVEL(1, "Incorrect parameters\n"); - if (displayLevel >= 1) usage(); - exit(1); -} - - -static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } -static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; } -static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } - - -int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) -{ - - if (!strcmp (input_filename, stdinmark)) - { - DISPLAYLEVEL(4,"Using stdin for input\n"); - *pfinput = stdin; - SET_BINARY_MODE(stdin); - } - else - { - *pfinput = fopen(input_filename, "rb"); - } - - if (!strcmp (output_filename, stdoutmark)) - { - DISPLAYLEVEL(4,"Using stdout for output\n"); - *pfoutput = stdout; - SET_BINARY_MODE(stdout); - } - else - { - // Check if destination file already exists - *pfoutput=0; - if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); - if (*pfoutput!=0) - { - fclose(*pfoutput); - if (!overwrite) - { - char ch; - DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); - DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); - if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible - ch = (char)getchar(); - if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); - } - } - *pfoutput = fopen( output_filename, "wb" ); - } - - if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); - if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); - - return 0; -} - - - -int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel) -{ - int (*compressionFunction)(const char*, char*, int); - unsigned long long filesize = 0; - unsigned long long compressedfilesize = MAGICNUMBER_SIZE; - char* in_buff; - char* out_buff; - FILE* finput; - FILE* foutput; - int displayLevel = (compressionlevel>0); - clock_t start, end; - size_t sizeCheck; - - - // Init - if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; - start = clock(); - get_fileHandle(input_filename, output_filename, &finput, &foutput); - if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; - - // Allocate Memory - in_buff = (char*)malloc(LEGACY_BLOCKSIZE); - out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); - if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); - - // Write Archive Header - *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER); - sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); - if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); - - // Main Loop - while (1) - { - unsigned int outSize; - // Read Block - int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); - if( inSize<=0 ) break; - filesize += inSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - - // Compress Block - outSize = compressionFunction(in_buff, out_buff+4, inSize); - compressedfilesize += outSize+4; - DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - // Write Block - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); - if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); - } - - // Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close & Free - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; -} - - -int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) -{ - void* (*initFunction) (const char*); - int (*compressionFunction)(void*, const char*, char*, int, int); - char* (*translateFunction) (void*); - int (*freeFunction) (void*); - void* ctx; - unsigned long long filesize = 0; - unsigned long long compressedfilesize = 0; - unsigned int checkbits; - char* in_buff, *in_start, *in_end; - char* out_buff; - FILE* finput; - FILE* foutput; - clock_t start, end; - unsigned int blockSize, inputBufferSize; - size_t sizeCheck, header_size; - void* streamChecksumState=NULL; - - - // Init - start = clock(); - if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; - if (compressionlevel>=3) - { - initFunction = LZ4_createHC; - compressionFunction = LZ4_compressHC_limitedOutput_continue; - translateFunction = LZ4_slideInputBufferHC; - freeFunction = LZ4_freeHC; - } - else - { - initFunction = LZ4_create; - compressionFunction = LZ4_compress_limitedOutput_continue; - translateFunction = LZ4_slideInputBuffer; - freeFunction = LZ4_free; - } - get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); - - // Allocate Memory - inputBufferSize = blockSize + 64 KB; - if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE; - in_buff = (char*)malloc(inputBufferSize); - out_buff = (char*)malloc(blockSize+CACHELINE); - if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); - in_start = in_buff; in_end = in_buff + inputBufferSize; - if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); - ctx = initFunction(in_buff); - - // Write Archive Header - *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention - *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01') - *(out_buff+4) |= (blockIndependence & _1BIT) << 5; - *(out_buff+4) |= (blockChecksum & _1BIT) << 4; - *(out_buff+4) |= (streamChecksum & _1BIT) << 2; - *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4); - checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED); - checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); - *(out_buff+6) = (unsigned char) checkbits; - header_size = 7; - sizeCheck = fwrite(out_buff, 1, header_size, foutput); - if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); - compressedfilesize += header_size; - - // Main Loop - while (1) - { - unsigned int outSize; - unsigned int inSize; - // Read Block - if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx); - inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput); - if( inSize==0 ) break; // No more input : end of compression - filesize += inSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize); - - // Compress Block - outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1); - if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; - if (blockChecksum) compressedfilesize+=4; - DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - // Write Block - if (outSize > 0) - { - int sizeToWrite; - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - if (blockChecksum) - { - unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); - } - sizeToWrite = 4 + outSize + (4*blockChecksum); - sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); - if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); - - } - else // Copy Original - { - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); - sizeCheck = fwrite(in_start, 1, inSize, foutput); - if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block"); - if (blockChecksum) - { - unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); - } - } - in_start += inSize; - } - - // End of Stream mark - * (unsigned int*) out_buff = LZ4S_EOS; - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += 4; - if (streamChecksum) - { - unsigned int checksum = XXH32_digest(streamChecksumState); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); - compressedfilesize += 4; - } - - // Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close & Free - freeFunction(ctx); - free(in_buff); - free(out_buff); - fclose(finput); - fclose(foutput); - - return 0; -} - - -int compress_file(char* input_filename, char* output_filename, int compressionlevel) -{ - int (*compressionFunction)(const char*, char*, int, int); - unsigned long long filesize = 0; - unsigned long long compressedfilesize = 0; - unsigned int checkbits; - char* in_buff; - char* out_buff; - char* headerBuffer; - FILE* finput; - FILE* foutput; - clock_t start, end; - int blockSize; - size_t sizeCheck, header_size, readSize; - void* streamChecksumState=NULL; - - // Branch out - if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel); - - // Init - start = clock(); - if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; - if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput; - get_fileHandle(input_filename, output_filename, &finput, &foutput); - blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); - - // Allocate Memory - in_buff = (char*)malloc(blockSize); - out_buff = (char*)malloc(blockSize+CACHELINE); - headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE); - if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory"); - if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); - - // Write Archive Header - *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention - *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01') - *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5; - *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4; - *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2; - *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4); - checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED); - checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); - *(headerBuffer+6) = (unsigned char) checkbits; - header_size = 7; - - // Write header - sizeCheck = fwrite(headerBuffer, 1, header_size, foutput); - if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); - compressedfilesize += header_size; - - // read first block - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); - - // Main Loop - while (readSize>0) - { - unsigned int outSize; - - filesize += readSize; - DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); - if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize); - - // Compress Block - outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1); - if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; - if (blockChecksum) compressedfilesize+=4; - DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); - - // Write Block - if (outSize > 0) - { - int sizeToWrite; - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); - if (blockChecksum) - { - unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); - } - sizeToWrite = 4 + outSize + (4*blockChecksum); - sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); - if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); - } - else // Copy Original Uncompressed - { - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); - sizeCheck = fwrite(in_buff, 1, readSize, foutput); - if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block"); - if (blockChecksum) - { - unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); - } - } - - // Read next block - readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); - } - - // End of Stream mark - * (unsigned int*) out_buff = LZ4S_EOS; - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); - compressedfilesize += 4; - if (streamChecksum) - { - unsigned int checksum = XXH32_digest(streamChecksumState); - * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); - sizeCheck = fwrite(out_buff, 1, 4, foutput); - if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); - compressedfilesize += 4; - } - - // Close & Free - free(in_buff); - free(out_buff); - free(headerBuffer); - fclose(finput); - fclose(foutput); - - // Final Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - return 0; -} - - -unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) -{ - unsigned long long filesize = 0; - char* in_buff; - char* out_buff; - unsigned int blockSize; - - - // Allocate Memory - in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); - out_buff = (char*)malloc(LEGACY_BLOCKSIZE); - if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); - - // Main Loop - while (1) - { - int decodeSize; - size_t sizeCheck; - - // Block Size - sizeCheck = fread(&blockSize, 1, 4, finput); - if (sizeCheck==0) break; // Nothing to read : file read is completed - blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian - if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) - { // Cannot read next block : maybe new stream ? - fseek(finput, -4, SEEK_CUR); - break; - } - - // Read Block - sizeCheck = fread(in_buff, 1, blockSize, finput); - - // Decode Block - decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); - if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !"); - filesize += decodeSize; - - // Write Block - sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); - if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n"); - } - - // Free - free(in_buff); - free(out_buff); - - return filesize; -} - - -unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) -{ - unsigned long long filesize = 0; - char* in_buff; - char* out_buff, *out_start, *out_end; - unsigned char descriptor[LZ4S_MAXHEADERSIZE]; - size_t nbReadBytes; - int decodedBytes=0; - unsigned int maxBlockSize; - size_t sizeCheck; - int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; - void* streamChecksumState=NULL; - int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe; - unsigned int prefix64k = 0; - - // Decode stream descriptor - nbReadBytes = fread(descriptor, 1, 3, finput); - if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header"); - { - int version = (descriptor[0] >> 6) & _2BITS; - int streamSize = (descriptor[0] >> 3) & _1BIT; - int reserved1 = (descriptor[0] >> 1) & _1BIT; - int dictionary = (descriptor[0] >> 0) & _1BIT; - - int reserved2 = (descriptor[1] >> 7) & _1BIT; - int blockSizeId = (descriptor[1] >> 4) & _3BITS; - int reserved3 = (descriptor[1] >> 0) & _4BITS; - int checkBits = (descriptor[2] >> 0) & _8BITS; - int checkBits_xxh32; - - blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT; - blockChecksumFlag = (descriptor[0] >> 4) & _1BIT; - streamChecksumFlag= (descriptor[0] >> 2) & _1BIT; - - if (version != 1) EXM_THROW(62, "Wrong version number"); - if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); - if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits"); - if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); - if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits"); - if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); - if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits"); - maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId); - // Checkbits verification - descriptor[1] &= 0xF0; - checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED); - checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32); - if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); - } - - if (!blockIndependenceFlag) - { - decompressionFunction = LZ4_decompress_safe_withPrefix64k; - prefix64k = 64 KB; - } - - // Allocate Memory - { - unsigned int outbuffSize = prefix64k+maxBlockSize; - in_buff = (char*)malloc(maxBlockSize); - if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE; - out_buff = (char*)malloc(outbuffSize); - out_end = out_buff + outbuffSize; - out_start = out_buff + prefix64k; - if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); - } - if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); - - // Main Loop - while (1) - { - unsigned int blockSize, uncompressedFlag; - - // Block Size - nbReadBytes = fread(&blockSize, 1, 4, finput); - if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size"); - if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed - blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian - uncompressedFlag = blockSize >> 31; - blockSize &= 0x7FFFFFFF; - if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size"); - - // Read Block - nbReadBytes = fread(in_buff, 1, blockSize, finput); - if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" ); - - // Check Block - if (blockChecksumFlag) - { - unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED); - unsigned int readChecksum; - sizeCheck = fread(&readChecksum, 1, 4, finput); - if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size"); - readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian - if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected"); - } - - if (uncompressedFlag) - { - // Write uncompressed Block - sizeCheck = fwrite(in_buff, 1, blockSize, foutput); - if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); - filesize += blockSize; - if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); - if (!blockIndependenceFlag) - { - if (blockSize >= prefix64k) - { - memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks - out_start = out_buff + prefix64k; - continue; - } - else - { - memcpy(out_start, in_buff, blockSize); - decodedBytes = blockSize; - } - } - } - else - { - // Decode Block - decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize); - if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); - filesize += decodedBytes; - if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); - - // Write Block - sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); - if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); - } - - if (!blockIndependenceFlag) - { - out_start += decodedBytes; - if ((size_t)(out_end - out_start) < (size_t)maxBlockSize) - { - memcpy(out_buff, out_start - prefix64k, prefix64k); - out_start = out_buff + prefix64k; - } - } - } - - // Stream Checksum - if (streamChecksumFlag) - { - unsigned int checksum = XXH32_digest(streamChecksumState); - unsigned int readChecksum; - sizeCheck = fread(&readChecksum, 1, 4, finput); - if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); - readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian - if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected"); - } - - // Free - free(in_buff); - free(out_buff); - - return filesize; -} - - -unsigned long long selectDecoder( FILE* finput, FILE* foutput) -{ - unsigned int magicNumber, size; - int errorNb; - size_t nbReadBytes; - - // Check Archive Header - nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput); - if (nbReadBytes==0) return 0; // EOF - if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable"); - magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format - if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers - - switch(magicNumber) - { - case LZ4S_MAGICNUMBER: - return DEFAULT_DECOMPRESSOR(finput, foutput); - case LEGACY_MAGICNUMBER: - DISPLAYLEVEL(4, "Detected : Legacy format \n"); - return decodeLegacyStream(finput, foutput); - case LZ4S_SKIPPABLE0: - DISPLAYLEVEL(4, "Skipping detected skippable area \n"); - nbReadBytes = fread(&size, 1, 4, finput); - if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); - size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format - errorNb = fseek(finput, size, SEEK_CUR); - if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); - return selectDecoder(finput, foutput); - EXTENDED_FORMAT; - default: - if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream - DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); - return 0; - } -} - - -int decodeFile(char* input_filename, char* output_filename) -{ - unsigned long long filesize = 0, decodedSize=0; - FILE* finput; - FILE* foutput; - clock_t start, end; - - - // Init - start = clock(); - get_fileHandle(input_filename, output_filename, &finput, &foutput); - - // Loop over multiple streams - do - { - decodedSize = selectDecoder(finput, foutput); - filesize += decodedSize; - } while (decodedSize); - - // Final Status - end = clock(); - DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); - { - double seconds = (double)(end - start)/CLOCKS_PER_SEC; - DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); - } - - // Close - fclose(finput); - fclose(foutput); - - // Error status = OK - return 0; -} - - -void waitEnter() -{ - DISPLAY("Press enter to continue...\n"); - getchar(); -} - - -int main(int argc, char** argv) -{ - int i, - cLevel=0, - decode=0, - bench=0, - filenamesStart=2, - legacy_format=0, - forceStdout=0, - forceCompress=0, - pause=0; - char* input_filename=0; - char* output_filename=0; - char* dynNameSpace=0; - char nullOutput[] = NULL_OUTPUT; - char extension[] = LZ4_EXTENSION; - - // Init - programName = argv[0]; - - for(i=1; i='1') && (argument[1] <='9')) - { - int iters = argument[1] - '0'; - BMK_SetNbIterations(iters); - argument++; - } - break; - - // Pause at the end (hidden option) - case 'p': pause=1; BMK_SetPause(); break; - - EXTENDED_ARGUMENTS; - - // Unrecognised command - default : badusage(); - } - } - continue; - } - - // first provided filename is input - if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } - - // second provided filename is output - if (!output_filename) - { - output_filename=argument; - if (!strcmp (output_filename, nullOutput)) output_filename = nulmark; - continue; - } - } - - DISPLAYLEVEL(3, WELCOME_MESSAGE); - DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2))); - - // No input filename ==> use stdin - if(!input_filename) { input_filename=stdinmark; } - - // Check if input or output are defined as console; trigger an error in this case - if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); - - // Check if benchmark is selected - if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); - - // No output filename ==> try to select one automatically (when possible) - while (!output_filename) - { - if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } // Default to stdout whenever possible (i.e. not a console) - if ((!decode) && !(forceCompress)) // auto-determine compression or decompression, based on file extension - { - size_t l = strlen(input_filename); - if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1; - } - if (!decode) // compression to file - { - size_t l = strlen(input_filename); - dynNameSpace = (char*)calloc(1,l+5); - output_filename = dynNameSpace; - strcpy(output_filename, input_filename); - strcpy(output_filename+l, LZ4_EXTENSION); - DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); - break; - } - // decompression to file (automatic name will work only if input filename has correct format extension) - { - size_t outl; - size_t inl = strlen(input_filename); - dynNameSpace = (char*)calloc(1,inl+1); - output_filename = dynNameSpace; - strcpy(output_filename, input_filename); - outl = inl; - if (inl>4) - while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0; - if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); } - DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); - } - } - - // No warning message in pure pipe mode (stdin + stdout) - if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; - - // Check if input or output are defined as console; trigger an error in this case - if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); - if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); - - // Decompress input if selected - if (decode) decodeFile(input_filename, output_filename); - else - // compression is default action - { - if (legacy_format) - { - DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n"); - legacy_compress_file(input_filename, output_filename, cLevel); - } - else - { - DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); - } - } - - if (pause) waitEnter(); - free(dynNameSpace); - return 0; -} diff --git a/programs/COPYING b/programs/COPYING new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/programs/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/programs/Makefile b/programs/Makefile new file mode 100644 index 0000000..2bc2902 --- /dev/null +++ b/programs/Makefile @@ -0,0 +1,100 @@ +# ################################################################ +# LZ4 programs - Makefile +# Copyright (C) Yann Collet 2011-2014 +# GPL v2 License +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# You can contact the author at : +# - LZ4 source repository : http://code.google.com/p/lz4/ +# - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c +# ################################################################ +# lz4 : Command Line Utility, supporting gzip-like arguments +# lz4c : CLU, supporting also legacy lz4demo arguments +# lz4c32: Same as lz4c, but forced to compile in 32-bits mode +# fuzzer : Test tool, to check lz4 integrity on target platform +# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# fullbench : Precisely measure speed for each LZ4 function variant +# fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# ################################################################ + +RELEASE=r111 +DESTDIR= +PREFIX=/usr +CC=gcc +CFLAGS+= -I. -std=c99 -Wall -W -Wundef -DLZ4_VERSION=\"$(RELEASE)\" + +BINDIR=$(PREFIX)/bin +MANDIR=$(PREFIX)/share/man/man1 +LZ4DIR=.. + + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + + +default: lz4 lz4c + +all: lz4 lz4c lz4c32 fuzzer fuzzer32 fullbench fullbench32 + +lz4: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c + $(CC) -O3 $(CFLAGS) -DDISABLE_LZ4C_LEGACY_OPTIONS $^ -o $@$(EXT) + +lz4c : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c + $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) + +lz4c32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c bench.c xxhash.c lz4cli.c + $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) + +fuzzer : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c + @echo fuzzer is a test tool to check lz4 integrity on target platform + $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) + +fuzzer32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c fuzzer.c + $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) + +fullbench : $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c + $(CC) -O3 $(CFLAGS) $^ -o $@$(EXT) + +fullbench32: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c xxhash.c fullbench.c + $(CC) -m32 -O3 $(CFLAGS) $^ -o $@$(EXT) + +clean: + @rm -f core *.o \ + lz4$(EXT) lz4c$(EXT) lz4c32$(EXT) \ + fuzzer$(EXT) fuzzer32$(EXT) fullbench$(EXT) fullbench32$(EXT) + @echo Cleaning completed + + +ifneq (,$(filter $(shell uname),Linux Darwin)) + +install: lz4 lz4c + @install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ + @install -m 755 lz4 $(DESTDIR)$(BINDIR)/lz4 + @install -m 755 lz4c $(DESTDIR)$(BINDIR)/lz4c + @install -m 644 lz4.1 $(DESTDIR)$(MANDIR)/lz4.1 + @echo lz4 installation completed + +uninstall: + [ -x $(DESTDIR)$(BINDIR)/lz4 ] && rm -f $(DESTDIR)$(BINDIR)/lz4 + [ -x $(DESTDIR)$(BINDIR)/lz4c ] && rm -f $(DESTDIR)$(BINDIR)/lz4c + [ -f $(DESTDIR)$(MANDIR)/lz4.1 ] && rm -f $(DESTDIR)$(MANDIR)/lz4.1 + @echo lz4 successfully uninstalled + +endif diff --git a/programs/bench.c b/programs/bench.c new file mode 100644 index 0000000..df3c44a --- /dev/null +++ b/programs/bench.c @@ -0,0 +1,441 @@ +/* + bench.c - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_DEPRECATE // VS2005 + +// Unix Large Files support (>4GB) +#define _FILE_OFFSET_BITS 64 +#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions +# define _LARGEFILE_SOURCE +#elif ! defined(__LP64__) // No point defining Large file for 64 bit +# define _LARGEFILE64_SOURCE +#endif + +// S_ISREG & gettimeofday() are not supported by MSVC +#if defined(_MSC_VER) || defined(_WIN32) +# define BMK_LEGACY_TIMER 1 +#endif + + +//************************************** +// Includes +//************************************** +#include // malloc +#include // fprintf, fopen, ftello64 +#include // stat64 +#include // stat64 + +// Use ftime() if gettimeofday() is not available on your target +#if defined(BMK_LEGACY_TIMER) +# include // timeb, ftime +#else +# include // gettimeofday +#endif + +#include "lz4.h" +#define COMPRESSOR0 LZ4_compress +#include "lz4hc.h" +#define COMPRESSOR1 LZ4_compressHC +#define DEFAULTCOMPRESSOR COMPRESSOR0 + +#include "xxhash.h" + + +//************************************** +// Compiler specifics +//************************************** +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + +// GCC does not support _rotl outside of Windows +#if !defined(_WIN32) +# define _rotl(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +//************************************** +// Constants +//************************************** +#define NBLOOPS 3 +#define TIMELOOP 2000 + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +#define KNUTH 2654435761U +#define MAX_MEM (2 GB - 64 MB) +#define DEFAULT_CHUNKSIZE (4 MB) + + +//************************************** +// Local structures +//************************************** +struct chunkParameters +{ + U32 id; + char* origBuffer; + char* compressedBuffer; + int origSize; + int compressedSize; +}; + +struct compressionParameters +{ + int (*compressionFunction)(const char*, char*, int); + int (*decompressionFunction)(const char*, char*, int); +}; + + +//************************************** +// MACRO +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + + + +//************************************** +// Benchmark Parameters +//************************************** +static int chunkSize = DEFAULT_CHUNKSIZE; +static int nbIterations = NBLOOPS; +static int BMK_pause = 0; + +void BMK_SetBlocksize(int bsize) { chunkSize = bsize; } + +void BMK_SetNbIterations(int nbLoops) +{ + nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", nbIterations); +} + +void BMK_SetPause() { BMK_pause = 1; } + + +//********************************************************* +// Private functions +//********************************************************* + +#if defined(BMK_LEGACY_TIMER) + +static int BMK_GetMilliStart() +{ + // Based on Legacy ftime() + // Rolls over every ~ 12.1 days (0x100000/24/60/60) + // Use GetMilliSpan to correct for rollover + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + +#else + +static int BMK_GetMilliStart() +{ + // Based on newer gettimeofday() + // Use GetMilliSpan to correct for rollover + struct timeval tv; + int nCount; + gettimeofday(&tv, NULL); + nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); + return nCount; +} + +#endif + + +static int BMK_GetMilliSpan( int nTimeStart ) +{ + int nSpan = BMK_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t step = (64 MB); + BYTE* testmem=NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + while (!testmem) + { + requiredMem -= step; + testmem = (BYTE*) malloc ((size_t)requiredMem); + } + + free (testmem); + return (size_t) (requiredMem - step); +} + + +static U64 BMK_GetFileSize(char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + return (U64)statbuf.st_size; +} + + +//********************************************************* +// Public function +//********************************************************* + +int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) +{ + int fileIdx=0; + char* orig_buff; + struct compressionParameters compP; + int cfunctionId; + + U64 totals = 0; + U64 totalz = 0; + double totalc = 0.; + double totald = 0.; + + + // Init + if (cLevel <3) cfunctionId = 0; else cfunctionId = 1; + switch (cfunctionId) + { +#ifdef COMPRESSOR0 + case 0 : compP.compressionFunction = COMPRESSOR0; break; +#endif +#ifdef COMPRESSOR1 + case 1 : compP.compressionFunction = COMPRESSOR1; break; +#endif + default : compP.compressionFunction = DEFAULTCOMPRESSOR; + } + compP.decompressionFunction = LZ4_decompress_fast; + + // Loop for each file + while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) + { + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + } + + // Alloc + chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters)); + orig_buff = (char*)malloc((size_t )benchedSize); + nbChunks = (int) (benchedSize / chunkSize) + 1; + maxCompressedChunkSize = LZ4_compressBound(chunkSize); + compressedBuffSize = nbChunks * maxCompressedChunkSize; + compressedBuffer = (char*)malloc((size_t )compressedBuffSize); + + + if (!orig_buff || !compressedBuffer) + { + DISPLAY("\nError: not enough memory!\n"); + free(orig_buff); + free(compressedBuffer); + free(chunkP); + fclose(inFile); + return 12; + } + + // Init chunks data + { + int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressedBuffer; + for (i=0; i chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + + // Fill input buffer + DISPLAY("Loading %s... \r", inFileName); + readSize = fread(orig_buff, 1, benchedSize, inFile); + fclose(inFile); + + if (readSize != benchedSize) + { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(orig_buff); + free(compressedBuffer); + free(chunkP); + return 13; + } + + // Calculating input Checksum + crcOrig = XXH32(orig_buff, (unsigned int)benchedSize,0); + + + // Bench + { + int loopNb, chunkNb; + size_t cSize=0; + double fastestC = 100000000., fastestD = 100000000.; + double ratio=0.; + U32 crcCheck=0; + + DISPLAY("\r%79s\r", ""); + for (loopNb = 1; loopNb <= nbIterations; loopNb++) + { + int nbLoops; + int milliTime; + + // Compression + DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, inFileName, (int)benchedSize); + { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000.); + + // Decompression + { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + + // CRC Checking + crcCheck = XXH32(orig_buff, (unsigned int)benchedSize,0); + if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; } + } + + if (crcOrig==crcCheck) + { + if (ratio<100.) + DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + else + DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / fastestC / 1000., (double)benchedSize / fastestD / 1000.); + } + totals += benchedSize; + totalz += cSize; + totalc += fastestC; + totald += fastestD; + } + + free(orig_buff); + free(compressedBuffer); + free(chunkP); + } + + if (nbFiles > 1) + DISPLAY("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.); + + if (BMK_pause) { DISPLAY("\npress enter...\n"); getchar(); } + + return 0; +} + + + diff --git a/programs/bench.h b/programs/bench.h new file mode 100644 index 0000000..ed801d4 --- /dev/null +++ b/programs/bench.h @@ -0,0 +1,41 @@ +/* + bench.h - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2013 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel); + +// Parameters +void BMK_SetBlocksize(int bsize); +void BMK_SetNbIterations(int nbLoops); +void BMK_SetPause(); + + + +#if defined (__cplusplus) +} +#endif diff --git a/programs/fullbench.c b/programs/fullbench.c new file mode 100644 index 0000000..c465c88 --- /dev/null +++ b/programs/fullbench.c @@ -0,0 +1,742 @@ +/* + bench.c - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_DEPRECATE // VS2005 + +// Unix Large Files support (>4GB) +#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions +# define _LARGEFILE_SOURCE +# define _FILE_OFFSET_BITS 64 +#elif ! defined(__LP64__) // No point defining Large file for 64 bit +# define _LARGEFILE64_SOURCE +#endif + +// S_ISREG & gettimeofday() are not supported by MSVC +#if defined(_MSC_VER) || defined(_WIN32) +# define BMK_LEGACY_TIMER 1 +#endif + + +//************************************** +// Includes +//************************************** +#include // malloc +#include // fprintf, fopen, ftello64 +#include // stat64 +#include // stat64 + +// Use ftime() if gettimeofday() is not available on your target +#if defined(BMK_LEGACY_TIMER) +# include // timeb, ftime +#else +# include // gettimeofday +#endif + +#include "lz4.h" +#define COMPRESSOR0 LZ4_compress +#include "lz4hc.h" +#define COMPRESSOR1 LZ4_compressHC +#define DEFAULTCOMPRESSOR COMPRESSOR0 + +#include "xxhash.h" + + +//************************************** +// Compiler Options +//************************************** +// S_ISREG & gettimeofday() are not supported by MSVC +#if !defined(S_ISREG) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +#endif + +// GCC does not support _rotl outside of Windows +#if !defined(_WIN32) +# define _rotl(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +//**************************** +// Constants +//**************************** +#define PROGRAM_DESCRIPTION "LZ4 speed analyzer" +#ifndef LZ4_VERSION +# define LZ4_VERSION "" +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__ + +#define NBLOOPS 6 +#define TIMELOOP 2500 + +#define KNUTH 2654435761U +#define MAX_MEM (1984<<20) +#define DEFAULT_CHUNKSIZE (4<<20) + +#define ALL_COMPRESSORS -1 +#define ALL_DECOMPRESSORS -1 + + +//************************************** +// Local structures +//************************************** +struct chunkParameters +{ + U32 id; + char* origBuffer; + char* compressedBuffer; + int origSize; + int compressedSize; +}; + + +//************************************** +// MACRO +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + + + +//************************************** +// Benchmark Parameters +//************************************** +static int chunkSize = DEFAULT_CHUNKSIZE; +static int nbIterations = NBLOOPS; +static int BMK_pause = 0; +static int compressionTest = 1; +static int decompressionTest = 1; +static int compressionAlgo = ALL_COMPRESSORS; +static int decompressionAlgo = ALL_DECOMPRESSORS; + +void BMK_SetBlocksize(int bsize) +{ + chunkSize = bsize; + DISPLAY("-Using Block Size of %i KB-\n", chunkSize>>10); +} + +void BMK_SetNbIterations(int nbLoops) +{ + nbIterations = nbLoops; + DISPLAY("- %i iterations -\n", nbIterations); +} + +void BMK_SetPause() +{ + BMK_pause = 1; +} + +//********************************************************* +// Private functions +//********************************************************* + +#if defined(BMK_LEGACY_TIMER) + +static int BMK_GetMilliStart() +{ + // Based on Legacy ftime() + // Rolls over every ~ 12.1 days (0x100000/24/60/60) + // Use GetMilliSpan to correct for rollover + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + +#else + +static int BMK_GetMilliStart() +{ + // Based on newer gettimeofday() + // Use GetMilliSpan to correct for rollover + struct timeval tv; + int nCount; + gettimeofday(&tv, NULL); + nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); + return nCount; +} + +#endif + + +static int BMK_GetMilliSpan( int nTimeStart ) +{ + int nSpan = BMK_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t step = (64U<<20); // 64 MB + BYTE* testmem=NULL; + + requiredMem = (((requiredMem >> 25) + 1) << 26); + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + requiredMem += 2*step; + while (!testmem) + { + requiredMem -= step; + testmem = (BYTE*) malloc ((size_t)requiredMem); + } + + free (testmem); + return (size_t) (requiredMem - step); +} + + +static U64 BMK_GetFileSize(char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + return (U64)statbuf.st_size; +} + + +//********************************************************* +// Public function +//********************************************************* + +static inline int local_LZ4_compress_limitedOutput(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); +} + +static void* stateLZ4; +static inline int local_LZ4_compress_withState(const char* in, char* out, int inSize) +{ + return LZ4_compress_withState(stateLZ4, in, out, inSize); +} + +static inline int local_LZ4_compress_limitedOutput_withState(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput_withState(stateLZ4, in, out, inSize, LZ4_compressBound(inSize)); +} + +static void* ctx; +static inline int local_LZ4_compress_continue(const char* in, char* out, int inSize) +{ + return LZ4_compress_continue(ctx, in, out, inSize); +} + +static inline int local_LZ4_compress_limitedOutput_continue(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); +} + +static void* stateLZ4HC; +static inline int local_LZ4_compressHC_withStateHC(const char* in, char* out, int inSize) +{ + return LZ4_compress_withState(stateLZ4HC, in, out, inSize); +} + +static inline int local_LZ4_compressHC_limitedOutput_withStateHC(const char* in, char* out, int inSize) +{ + return LZ4_compress_limitedOutput_withState(stateLZ4HC, in, out, inSize, LZ4_compressBound(inSize)); +} + +static inline int local_LZ4_compressHC_limitedOutput(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput(in, out, inSize, LZ4_compressBound(inSize)); +} + +static inline int local_LZ4_compressHC_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_continue(ctx, in, out, inSize); +} + +static inline int local_LZ4_compressHC_limitedOutput_continue(const char* in, char* out, int inSize) +{ + return LZ4_compressHC_limitedOutput_continue(ctx, in, out, inSize, LZ4_compressBound(inSize)); +} + +static inline int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize) +{ + (void)inSize; + LZ4_decompress_fast(in, out, outSize); + return outSize; +} + +static inline int local_LZ4_decompress_fast_withPrefix64k(const char* in, char* out, int inSize, int outSize) +{ + (void)inSize; + LZ4_decompress_fast_withPrefix64k(in, out, outSize); + return outSize; +} + +static inline int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize) +{ + return LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize); +} + +int fullSpeedBench(char** fileNamesTable, int nbFiles) +{ + int fileIdx=0; + char* orig_buff; +# define NB_COMPRESSION_ALGORITHMS 12 +# define MINCOMPRESSIONCHAR '0' +# define MAXCOMPRESSIONCHAR (MINCOMPRESSIONCHAR + NB_COMPRESSION_ALGORITHMS) + static char* compressionNames[] = { "LZ4_compress", "LZ4_compress_limitedOutput", + "LZ4_compress_withState", "LZ4_compress_limitedOutput_withState", + "LZ4_compress_continue", "LZ4_compress_limitedOutput_continue", + "LZ4_compressHC", "LZ4_compressHC_limitedOutput", + "LZ4_compressHC_withStateHC", "LZ4_compressHC_limitedOutput_withStateHC", + "LZ4_compressHC_continue", "LZ4_compressHC_limitedOutput_continue" }; + double totalCTime[NB_COMPRESSION_ALGORITHMS] = {0}; + double totalCSize[NB_COMPRESSION_ALGORITHMS] = {0}; +# define NB_DECOMPRESSION_ALGORITHMS 5 +# define MINDECOMPRESSIONCHAR '0' +# define MAXDECOMPRESSIONCHAR (MINDECOMPRESSIONCHAR + NB_DECOMPRESSION_ALGORITHMS) + static char* decompressionNames[] = { "LZ4_decompress_fast", "LZ4_decompress_fast_withPrefix64k", "LZ4_decompress_safe", "LZ4_decompress_safe_withPrefix64k", "LZ4_decompress_safe_partial" }; + double totalDTime[NB_DECOMPRESSION_ALGORITHMS] = {0}; + + U64 totals = 0; + + + // Loop for each file + while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) + { + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + } + + // Alloc + chunkP = (struct chunkParameters*) malloc(((benchedSize / chunkSize)+1) * sizeof(struct chunkParameters)); + orig_buff = (char*) malloc((size_t)benchedSize); + nbChunks = (int) (benchedSize / chunkSize) + 1; + maxCompressedChunkSize = LZ4_compressBound(chunkSize); + compressedBuffSize = nbChunks * maxCompressedChunkSize; + compressed_buff = (char*)malloc((size_t)compressedBuffSize); + + + if(!orig_buff || !compressed_buff) + { + DISPLAY("\nError: not enough memory!\n"); + free(orig_buff); + free(compressed_buff); + free(chunkP); + fclose(inFile); + return 12; + } + + // Init chunks data + { + int i; + size_t remaining = benchedSize; + char* in = orig_buff; + char* out = compressed_buff; + for (i=0; i chunkSize) { chunkP[i].origSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; } + chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize; + chunkP[i].compressedSize = 0; + } + } + + // Fill input buffer + DISPLAY("Loading %s... \r", inFileName); + readSize = fread(orig_buff, 1, benchedSize, inFile); + fclose(inFile); + + if(readSize != benchedSize) + { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(orig_buff); + free(compressed_buff); + free(chunkP); + return 13; + } + + // Calculating input Checksum + crcOriginal = XXH32(orig_buff, (unsigned int)benchedSize,0); + + + // Bench + { + int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb; + size_t cSize=0; + double ratio=0.; + + DISPLAY("\r%79s\r", ""); + DISPLAY(" %s : \n", inFileName); + + // Compression Algorithms + for (cAlgNb=0; (cAlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); cAlgNb++) + { + char* cName = compressionNames[cAlgNb]; + int (*compressionFunction)(const char*, char*, int); + void* (*initFunction)(const char*) = NULL; + double bestTime = 100000000.; + + if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != cAlgNb)) continue; + + switch(cAlgNb) + { + case 0 : compressionFunction = LZ4_compress; break; + case 1 : compressionFunction = local_LZ4_compress_limitedOutput; break; + case 2 : compressionFunction = local_LZ4_compress_withState; break; + case 3 : compressionFunction = local_LZ4_compress_limitedOutput_withState; break; + case 4 : compressionFunction = local_LZ4_compress_continue; initFunction = LZ4_create; break; + case 5 : compressionFunction = local_LZ4_compress_limitedOutput_continue; initFunction = LZ4_create; break; + case 6 : compressionFunction = LZ4_compressHC; break; + case 7 : compressionFunction = local_LZ4_compressHC_limitedOutput; break; + case 8 : compressionFunction = local_LZ4_compressHC_withStateHC; break; + case 9 : compressionFunction = local_LZ4_compressHC_limitedOutput_withStateHC; break; + case 10: compressionFunction = local_LZ4_compressHC_continue; initFunction = LZ4_createHC; break; + case 11: compressionFunction = local_LZ4_compressHC_limitedOutput_continue; initFunction = LZ4_createHC; break; + default : DISPLAY("ERROR ! Bad algorithm Id !! \n"); free(chunkP); return 1; + } + + for (loopNb = 1; loopNb <= nbIterations; loopNb++) + { + double averageTime; + int milliTime; + + DISPLAY("%1i-%-19.19s : %9i ->\r", loopNb, cName, (int)benchedSize); + { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s\r", loopNb, cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + } + + if (ratio<100.) + DISPLAY("%-21.21s : %9i -> %9i (%5.2f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + else + DISPLAY("%-21.21s : %9i -> %9i (%5.1f%%),%7.1f MB/s\n", cName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000.); + + totalCTime[cAlgNb] += bestTime; + totalCSize[cAlgNb] += cSize; + } + + // Prepare layout for decompression + for (chunkNb=0; chunkNb\r", loopNb, dName, (int)benchedSize); + + nb_loops = 0; + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + for (chunkNb=0; chunkNb %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); + + // CRC Checking + crcDecoded = XXH32(orig_buff, (int)benchedSize, 0); + if (crcOriginal!=crcDecoded) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded); exit(1); } + } + + DISPLAY("%-26.26s :%10i -> %7.1f MB/s\n", dName, (int)benchedSize, (double)benchedSize / bestTime / 1000.); + + totalDTime[dAlgNb] += bestTime; + } + + totals += benchedSize; + } + + free(orig_buff); + free(compressed_buff); + free(chunkP); + } + + if (nbFiles > 1) + { + int AlgNb; + + DISPLAY(" ** TOTAL ** : \n"); + for (AlgNb = 0; (AlgNb < NB_COMPRESSION_ALGORITHMS) && (compressionTest); AlgNb ++) + { + char* cName = compressionNames[AlgNb]; + if ((compressionAlgo != ALL_COMPRESSORS) && (compressionAlgo != AlgNb)) continue; + DISPLAY("%-21.21s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s\n", cName, (long long unsigned int)totals, (long long unsigned int)totalCSize[AlgNb], (double)totalCSize[AlgNb]/(double)totals*100., (double)totals/totalCTime[AlgNb]/1000.); + } + for (AlgNb = 0; (AlgNb < NB_DECOMPRESSION_ALGORITHMS) && (decompressionTest); AlgNb ++) + { + char* dName = decompressionNames[AlgNb]; + if ((decompressionAlgo != ALL_DECOMPRESSORS) && (decompressionAlgo != AlgNb)) continue; + DISPLAY("%-21.21s :%10llu -> %6.1f MB/s\n", dName, (long long unsigned int)totals, (double)totals/totalDTime[AlgNb]/1000.); + } + } + + if (BMK_pause) { printf("press enter...\n"); getchar(); } + + return 0; +} + + +int usage(char* exename) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " -c : compression tests only\n"); + DISPLAY( " -d : decompression tests only\n"); + DISPLAY( " -H/-h : Help (this text + advanced options)\n"); + return 0; +} + +int usage_advanced() +{ + DISPLAY( "\nAdvanced options :\n"); + DISPLAY( " -c# : test only compression function # [%c-%c]\n", MINCOMPRESSIONCHAR, MAXCOMPRESSIONCHAR); + DISPLAY( " -d# : test only compression function # [%c-%c]\n", MINDECOMPRESSIONCHAR, MAXDECOMPRESSIONCHAR); + DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); + DISPLAY( " -B# : Block size [4-7](default : 7)\n"); + //DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + return 0; +} + +int badusage(char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 0; +} + +int main(int argc, char** argv) +{ + int i, + filenamesStart=2; + char* exename=argv[0]; + char* input_filename=0; + + // Welcome message + DISPLAY( WELCOME_MESSAGE); + + if (argc<2) { badusage(exename); return 1; } + + for(i=1; i= MINCOMPRESSIONCHAR) && (argument[1]<= MAXCOMPRESSIONCHAR)) + compressionAlgo = argument[1] - '0', argument++; + break; + + // Select decompression algorithm only + case 'd': + compressionTest = 0; + if ((argument[1]>= MINDECOMPRESSIONCHAR) && (argument[1]<= MAXDECOMPRESSIONCHAR)) + decompressionAlgo = argument[1] - '0', argument++; + break; + + // Display help on usage + case 'h' : + case 'H': usage(exename); usage_advanced(); return 0; + + // Modify Block Properties + case 'B': + while (argument[1]!=0) + switch(argument[1]) + { + case '4': + case '5': + case '6': + case '7': + { + int B = argument[1] - '0'; + int S = 1 << (8 + 2*B); + BMK_SetBlocksize(S); + argument++; + break; + } + case 'D': argument++; break; + default : goto _exit_blockProperties; + } +_exit_blockProperties: + break; + + // Modify Nb Iterations + case 'i': + if ((argument[1] >='1') && (argument[1] <='9')) + { + int iters = argument[1] - '0'; + BMK_SetNbIterations(iters); + argument++; + } + break; + + // Pause at the end (hidden option) + case 'p': BMK_SetPause(); break; + + // Unrecognised command + default : badusage(exename); return 1; + } + } + continue; + } + + // first provided filename is input + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + + } + + // No input filename ==> Error + if(!input_filename) { badusage(exename); return 1; } + + return fullSpeedBench(argv+filenamesStart, argc-filenamesStart); + +} + diff --git a/programs/fuzzer.c b/programs/fuzzer.c new file mode 100644 index 0000000..4513ebe --- /dev/null +++ b/programs/fuzzer.c @@ -0,0 +1,303 @@ +/* + fuzzer.c - Fuzzer test tool for LZ4 + Copyright (C) Yann Collet - Andrew Mahone 2012-2013 + Code started by Andrew Mahone, modified by Yann Collet + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Remove Visual warning messages +//************************************** +#define _CRT_SECURE_NO_WARNINGS // fgets + + +//************************************** +// Includes +//************************************** +#include +#include // fgets, sscanf +#include // timeb +#include "lz4.h" +#include "lz4hc.h" + + +//************************************** +// Constants +//************************************** +#ifndef LZ4_VERSION +# define LZ4_VERSION "" +#endif + +#define NB_ATTEMPTS (1<<17) +#define LEN ((1<<15)) +#define SEQ_POW 2 +#define NUM_SEQ (1 << SEQ_POW) +#define SEQ_MSK ((NUM_SEQ) - 1) +#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0) +#define NEW_SEQ(x) ((((x) >> 10) %10) == 0) +#define PAGE_SIZE 4096 +#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) +#define PRIME1 2654435761U +#define PRIME2 2246822519U +#define PRIME3 3266489917U + + +//********************************************************* +// Functions +//********************************************************* +static int FUZ_GetMilliStart() +{ + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + + +static int FUZ_GetMilliSpan( int nTimeStart ) +{ + int nSpan = FUZ_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +unsigned int FUZ_rand(unsigned int* src) +{ + *src = ((*src) * PRIME1) + PRIME2; + return *src; +} + + +int test_canary(unsigned char *buf) +{ + int i; + for (i = 0; i < 2048; i++) + if (buf[i] != buf[i + 2048]) + return 0; + return 1; +} + + +int FUZ_SecurityTest() +{ + char* output; + char* input; + int i, r; + + printf("Overflow test (issue 52)...\n"); + input = (char*) malloc (20<<20); + output = (char*) malloc (20<<20); + input[0] = 0x0F; + input[1] = 0x00; + input[2] = 0x00; + for(i = 3; i < 16840000; i++) + input[i] = 0xff; + r = LZ4_decompress_fast(input, output, 20<<20); + + free(input); + free(output); + printf(" Passed (return = %i < 0)\n",r); + return 0; +} + + +//int main(int argc, char *argv[]) { +int main() { + unsigned long long bytes = 0; + unsigned long long cbytes = 0; + unsigned long long hcbytes = 0; + unsigned char buf[LEN]; + unsigned char testOut[LEN+1]; +# define FUZ_max LZ4_COMPRESSBOUND(LEN) +# define FUZ_avail ROUND_PAGE(FUZ_max) + const int off_full = FUZ_avail - FUZ_max; + unsigned char cbuf[FUZ_avail + PAGE_SIZE]; + unsigned int seed, randState, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); + int i, j, k, ret, len, lenHC, attemptNb; + char userInput[30] = {0}; +# define FUZ_CHECKTEST(cond, message) if (cond) { printf("Test %i : %s : seed %u, cycle %i \n", testNb, message, seed, attemptNb); goto _output_error; } +# define FUZ_DISPLAYTEST testNb++; printf("%2i\b\b", testNb); + void* stateLZ4 = malloc(LZ4_sizeofState()); + void* stateLZ4HC = malloc(LZ4_sizeofStateHC()); + + printf("starting LZ4 fuzzer (%s)\n", LZ4_VERSION); + printf("Select an Initialisation number (default : random) : "); + fflush(stdout); + if ( fgets(userInput, sizeof userInput, stdin) ) + { + if ( sscanf(userInput, "%d", &seed) == 1 ) {} + else seed = FUZ_GetMilliSpan(timestamp); + } + printf("Seed = %u\n", seed); + randState = seed; + + //FUZ_SecurityTest(); + + for (i = 0; i < 2048; i++) + cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&randState) >> 16; + + for (attemptNb = 0; attemptNb < NB_ATTEMPTS; attemptNb++) + { + int testNb = 0; + + printf("\r%7i /%7i - ", attemptNb, NB_ATTEMPTS); + + for (j = 0; j < NUM_SEQ; j++) { + seeds[j] = FUZ_rand(&randState) << 8; + seeds[j] ^= (FUZ_rand(&randState) >> 8) & 65535; + } + for (j = 0; j < LEN; j++) { + k = FUZ_rand(&randState); + if (j == 0 || NEW_SEQ(k)) + cur_seq = seeds[(FUZ_rand(&randState) >> 16) & SEQ_MSK]; + if (MOD_SEQ(k)) { + k = (FUZ_rand(&randState) >> 16) & SEQ_MSK; + seeds[k] = FUZ_rand(&randState) << 8; + seeds[k] ^= (FUZ_rand(&randState) >> 8) & 65535; + } + buf[j] = FUZ_rand(&cur_seq) >> 16; + } + + // Test compression HC + FUZ_DISPLAYTEST; // 1 + ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); + lenHC = ret; + + // Test compression HC using external state + FUZ_DISPLAYTEST; // 1 + ret = LZ4_compressHC_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[off_full], LEN); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed"); + + // Test compression using external state + FUZ_DISPLAYTEST; // 2 + ret = LZ4_compress_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN); + FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed"); + + // Test compression + FUZ_DISPLAYTEST; // 2 + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); + len = ret; + + // Test decoding with output size being exactly what's necessary => must work + FUZ_DISPLAYTEST; // 3 + ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space"); + + // Test decoding with one byte missing => must fail + FUZ_DISPLAYTEST; // 4 + ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN-1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small"); + + // Test decoding with one byte too much => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_fast((char*)&cbuf[off_full], (char*)testOut, LEN+1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large"); + + // Test decoding with enough output size => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); + + // Test decoding with output size being exactly what's necessary => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space"); + + // Test decoding with output size being one byte too short => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short"); + + // Test decoding with input size being one byte too short => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short"); + + // Test decoding with input size being one byte too large => must fail + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); + FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large"); + //if (ret>=0) { printf("Test 10 : decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test partial decoding with target output size being max/2 => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN/2, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); + + // Test partial decoding with target output size being just below max => must work + FUZ_DISPLAYTEST; + ret = LZ4_decompress_safe_partial((char*)&cbuf[off_full], (char*)testOut, len, LEN-3, LEN); + FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space"); + + // Test compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput() failed despite sufficient space"); + FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); + + // Test compression with output size being exactly what's necessary and external state (should work) + FUZ_DISPLAYTEST; // 2 + ret = LZ4_compress_limitedOutput_withState(stateLZ4, (const char*)buf, (char*)&cbuf[off_full], LEN, len); + FUZ_CHECKTEST(ret==0, "LZ4_compress_limitedOutput_withState() failed despite sufficient space"); + FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); + + // Test HC compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space"); + + // Test HC compression with output size being exactly what's necessary (should work) + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput_withStateHC(stateLZ4HC, (const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, lenHC); + FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space"); + + // Test compression with just one missing byte into output buffer => must fail + FUZ_DISPLAYTEST; + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); + FUZ_CHECKTEST(ret, "compression overran output buffer"); + FUZ_CHECKTEST(!test_canary(&cbuf[FUZ_avail]), "compression overran output buffer"); + + // Test HC compression with just one missing byte into output buffer => must fail + FUZ_DISPLAYTEST; + ret = LZ4_compressHC_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, lenHC-1); + FUZ_CHECKTEST(ret, "HC compression overran output buffer"); + + bytes += LEN; + cbytes += len; + hcbytes += lenHC; + FUZ_rand(&randState); + } + + printf("all tests completed successfully \n"); + printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); + printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100); + getchar(); + return 0; + +_output_error: + getchar(); + return 1; +} diff --git a/programs/lz4.1 b/programs/lz4.1 new file mode 100644 index 0000000..69c58c3 --- /dev/null +++ b/programs/lz4.1 @@ -0,0 +1,87 @@ +\" +\" lz4.1: This is a manual page for 'lz4' program. This file is part of the +\" lz4 project. +\" + +\" No hyphenation +.hy 0 +.nr HY 0 + +.TH lz4 man +.SH NAME +\fBlz4\fR - Extremely fast compression algorithm + +.SH SYNOPSIS +.TP 5 +\fBlz4\fR [\fBOPTIONS\fR] [-|INPUT-FILE] + +.SH DESCRIPTION +.PP +\fBlz4\fR is an extremely fast lossless compression algorithm. It is based on +the \fBLZ77\fR family of compression scheme. At the compression speed of 400 +MB/s per core, \fBlz4\fR is also scalable with multi-core CPUs. It features +an extremely fast decoder, with speed in multiple GB/s per core, typically +reaching the RAM speed limits on multi-core systems. \fBlz4\fR supports +following options + +.SH OPTIONS +.TP +.B \-1 + fast compression (default) +.TP +.B \-9 + high compression +.TP +.B \-d + decompression +.TP +.B \-f + overwrite output without prompting +.TP +.B \-h/\-H + display help/long help and exit +.TP +.B \-V + display Version number and exit +.TP +.B \-v + verbose mode +.TP +.B \-q + suppress warnings; specify twice to suppress errors too +.TP +.B \-c + force write to standard output, even if it is the console +.TP +.B \-t + test compressed file integrity +.TP +.B \-z + force compression +.TP +.B \-l + use Legacy format (useful for Linux Kernel compression) +.TP +.B \-B# + block size [4-7](default : 7) +.TP +.B \-BD + block dependency (improve compression ratio) +.TP +.B \-BX + enable block checksum (default:disabled) +.TP +.B \-Sx + disable stream checksum (default:enabled) +.TP +.B \-b + benchmark file(s) +.TP +.B \-i# + iteration loops [1-9](default : 3), benchmark mode only + +.SH BUGS +Report bugs at:- https://code.google.com/p/lz4/ + +.SH AUTHOR +Yann Collet \ No newline at end of file diff --git a/programs/lz4cli.c b/programs/lz4cli.c new file mode 100644 index 0000000..f69c8a0 --- /dev/null +++ b/programs/lz4cli.c @@ -0,0 +1,1257 @@ +/* + LZ4cli.c - LZ4 Command Line Interface + Copyright (C) Yann Collet 2011-2013 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 source repository : http://code.google.com/p/lz4/ + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +/* + Note : this is stand-alone program. + It is not part of LZ4 compression library, it is a user program of the LZ4 library. + The license of LZ4 library is BSD. + The license of xxHash library is BSD. + The license of this compression CLI program is GPLv2. +*/ + +//************************************** +// Tuning parameters +//************************************** +// DISABLE_LZ4C_LEGACY_OPTIONS : +// Control the availability of -c0, -c1 and -hc legacy arguments +// Default : Legacy options are enabled +// #define DISABLE_LZ4C_LEGACY_OPTIONS + + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# define _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_DEPRECATE // VS2005 +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#define _FILE_OFFSET_BITS 64 // Large file support on 32-bits unix +#define _POSIX_SOURCE 1 // for fileno() within on unix + + +//**************************** +// Includes +//**************************** +#include // fprintf, fopen, fread, _fileno, stdin, stdout +#include // malloc +#include // strcmp, strlen +#include // clock +#include "lz4.h" +#include "lz4hc.h" +#include "xxhash.h" +#include "bench.h" + + +//**************************** +// OS-specific Includes +//**************************** +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include // _O_BINARY +# include // _setmode, _isatty +# ifdef __MINGW32__ + int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into +# endif +# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# include // isatty +# define SET_BINARY_MODE(file) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#endif + + +//************************************** +// Compiler-specific functions +//************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#if defined(_MSC_VER) // Visual Studio +# define swap32 _byteswap_ulong +#elif GCC_VERSION >= 403 +# define swap32 __builtin_bswap32 +#else + static inline unsigned int swap32(unsigned int x) + { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); + } +#endif + + +//**************************** +// Constants +//**************************** +#define COMPRESSOR_NAME "LZ4 Compression CLI" +#ifndef LZ4_VERSION +# define LZ4_VERSION "v1.1.0" +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), LZ4_VERSION, AUTHOR, __DATE__ +#define LZ4_EXTENSION ".lz4" + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define MAGICNUMBER_SIZE 4 +#define LZ4S_MAGICNUMBER 0x184D2204 +#define LZ4S_SKIPPABLE0 0x184D2A50 +#define LZ4S_SKIPPABLEMASK 0xFFFFFFF0 +#define LEGACY_MAGICNUMBER 0x184C2102 + +#define CACHELINE 64 +#define LEGACY_BLOCKSIZE (8 MB) +#define MIN_STREAM_BUFSIZE (1 MB + 64 KB) +#define LZ4S_BLOCKSIZEID_DEFAULT 7 +#define LZ4S_CHECKSUM_SEED 0 +#define LZ4S_EOS 0 +#define LZ4S_MAXHEADERSIZE (MAGICNUMBER_SIZE+2+8+4+1) + + +//************************************** +// Architecture Macros +//************************************** +static const int one = 1; +#define CPU_LITTLE_ENDIAN (*(char*)(&one)) +#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) +#define LITTLE_ENDIAN_32(i) (CPU_LITTLE_ENDIAN?(i):swap32(i)) + + +//************************************** +// Macros +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } + + +//************************************** +// Special input/output +//************************************** +#define NULL_OUTPUT "null" +char stdinmark[] = "stdin"; +char stdoutmark[] = "stdout"; +#ifdef _WIN32 +char nulmark[] = "nul"; +#else +char nulmark[] = "/dev/null"; +#endif + + +//************************************** +// Local Parameters +//************************************** +static char* programName; +static int displayLevel = 2; // 0 : no display // 1: errors // 2 : + result + interaction + warnings ; // 3 : + progression; // 4 : + information +static int overwrite = 0; +static int blockSizeId = LZ4S_BLOCKSIZEID_DEFAULT; +static int blockChecksum = 0; +static int streamChecksum = 1; +static int blockIndependence = 1; + + +//************************************** +// Exceptions +//************************************** +#define DEBUG 0 +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +//************************************** +// Version modifiers +//************************************** +#define EXTENDED_ARGUMENTS +#define EXTENDED_HELP +#define EXTENDED_FORMAT +#define DEFAULT_COMPRESSOR compress_file +#define DEFAULT_DECOMPRESSOR decodeLZ4S + + +//**************************** +// Functions +//**************************** +int usage() +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] [input] [output]\n", programName); + DISPLAY( "\n"); + DISPLAY( "input : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - or %s, read standard input\n", stdinmark); + DISPLAY( "Arguments :\n"); + DISPLAY( " -1 : Fast compression (default) \n"); + DISPLAY( " -9 : High compression \n"); + DISPLAY( " -d : decompression (default for %s extension)\n", LZ4_EXTENSION); + DISPLAY( " -z : force compression\n"); + DISPLAY( " -f : overwrite output without prompting \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); + return 0; +} + +int usage_advanced() +{ + DISPLAY(WELCOME_MESSAGE); + usage(); + DISPLAY( "\n"); + DISPLAY( "Advanced arguments :\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -v : verbose mode\n"); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); + DISPLAY( " -c : force write to standard output, even if it is the console\n"); + DISPLAY( " -t : test compressed file integrity\n"); + DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n"); + DISPLAY( " -B# : Block size [4-7](default : 7)\n"); + DISPLAY( " -BD : Block dependency (improve compression ratio)\n"); + DISPLAY( " -BX : enable block checksum (default:disabled)\n"); + DISPLAY( " -Sx : disable stream checksum (default:enabled)\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b : benchmark file(s)\n"); + DISPLAY( " -i# : iteration loops [1-9](default : 3), benchmark mode only\n"); +#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) + DISPLAY( "Legacy arguments :\n"); + DISPLAY( " -c0 : fast compression\n"); + DISPLAY( " -c1 : high compression\n"); + DISPLAY( " -hc : high compression\n"); + DISPLAY( " -y : overwrite output without prompting \n"); + DISPLAY( " -s : suppress warnings \n"); +#endif // DISABLE_LZ4C_LEGACY_OPTIONS + EXTENDED_HELP; + return 0; +} + +int usage_longhelp() +{ + DISPLAY( "\n"); + DISPLAY( "Which values can get [output] ? \n"); + DISPLAY( "[output] : a filename\n"); + DISPLAY( " '%s', or '-' for standard output (pipe mode)\n", stdoutmark); + DISPLAY( " '%s' to discard output (test mode)\n", NULL_OUTPUT); + DISPLAY( "[output] can be left empty. In this case, it receives the following value : \n"); + DISPLAY( " - if stdout is not the console, then [output] = stdout \n"); + DISPLAY( " - if stdout is console : \n"); + DISPLAY( " + if compression selected, output to filename%s \n", LZ4_EXTENSION); + DISPLAY( " + if decompression selected, output to filename without '%s'\n", LZ4_EXTENSION); + DISPLAY( " > if input filename has no '%s' extension : error\n", LZ4_EXTENSION); + DISPLAY( "\n"); + DISPLAY( "Compression levels : \n"); + DISPLAY( "There are technically 2 accessible compression levels.\n"); + DISPLAY( "-0 ... -2 => Fast compression\n"); + DISPLAY( "-3 ... -9 => High compression\n"); + DISPLAY( "\n"); + DISPLAY( "stdin, stdout and the console : \n"); + DISPLAY( "To protect the console from binary flooding (bad argument mistake)\n"); + DISPLAY( "%s will refuse to read from console, or write to console \n", programName); + DISPLAY( "except if '-c' command is specified, to force output to console \n"); + DISPLAY( "\n"); + DISPLAY( "Simple example :\n"); + DISPLAY( "1 : compress 'filename' fast, using default output name 'filename.lz4'\n"); + DISPLAY( " %s filename\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments can be appended together, or provided independently. For example :\n"); + DISPLAY( "2 : compress 'filename' in high compression mode, overwrite output if exists\n"); + DISPLAY( " %s -f9 filename \n", programName); + DISPLAY( " is equivalent to :\n"); + DISPLAY( " %s -f -9 filename \n", programName); + DISPLAY( "\n"); + DISPLAY( "%s can be used in 'pure pipe mode', for example :\n", programName); + DISPLAY( "3 : compress data stream from 'generator', send result to 'consumer'\n"); + DISPLAY( " generator | %s | consumer \n", programName); +#if !defined(DISABLE_LZ4C_LEGACY_OPTIONS) + DISPLAY( "\n"); + DISPLAY( "Warning :\n"); + DISPLAY( "Legacy arguments take precedence. Therefore : \n"); + DISPLAY( " %s -hc filename\n", programName); + DISPLAY( "means 'compress filename in high compression mode'\n"); + DISPLAY( "It is not equivalent to :\n"); + DISPLAY( " %s -h -c filename\n", programName); + DISPLAY( "which would display help text and exit\n"); +#endif // DISABLE_LZ4C_LEGACY_OPTIONS + return 0; +} + +int badusage() +{ + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (displayLevel >= 1) usage(); + exit(1); +} + + +static int LZ4S_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } +static unsigned int LZ4S_GetCheckBits_FromXXH (unsigned int xxh) { return (xxh >> 8) & _8BITS; } +static int LZ4S_isSkippableMagicNumber(unsigned int magic) { return (magic & LZ4S_SKIPPABLEMASK) == LZ4S_SKIPPABLE0; } + + +int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) +{ + + if (!strcmp (input_filename, stdinmark)) + { + DISPLAYLEVEL(4,"Using stdin for input\n"); + *pfinput = stdin; + SET_BINARY_MODE(stdin); + } + else + { + *pfinput = fopen(input_filename, "rb"); + } + + if (!strcmp (output_filename, stdoutmark)) + { + DISPLAYLEVEL(4,"Using stdout for output\n"); + *pfoutput = stdout; + SET_BINARY_MODE(stdout); + } + else + { + // Check if destination file already exists + *pfoutput=0; + if (output_filename != nulmark) *pfoutput = fopen( output_filename, "rb" ); + if (*pfoutput!=0) + { + fclose(*pfoutput); + if (!overwrite) + { + char ch; + DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename); + DISPLAYLEVEL(2, "Overwrite ? (Y/N) : "); + if (displayLevel <= 1) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); // No interaction possible + ch = (char)getchar(); + if ((ch!='Y') && (ch!='y')) EXM_THROW(11, "Operation aborted : %s already exists", output_filename); + } + } + *pfoutput = fopen( output_filename, "wb" ); + } + + if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename); + if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename); + + return 0; +} + + + +int legacy_compress_file(char* input_filename, char* output_filename, int compressionlevel) +{ + int (*compressionFunction)(const char*, char*, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = MAGICNUMBER_SIZE; + char* in_buff; + char* out_buff; + FILE* finput; + FILE* foutput; + int displayLevel = (compressionlevel>0); + clock_t start, end; + size_t sizeCheck; + + + // Init + if (compressionlevel < 3) compressionFunction = LZ4_compress; else compressionFunction = LZ4_compressHC; + start = clock(); + get_fileHandle(input_filename, output_filename, &finput, &foutput); + if ((displayLevel==2) && (compressionlevel==1)) displayLevel=3; + + // Allocate Memory + in_buff = (char*)malloc(LEGACY_BLOCKSIZE); + out_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + if (!in_buff || !out_buff) EXM_THROW(21, "Allocation error : not enough memory"); + + // Write Archive Header + *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LEGACY_MAGICNUMBER); + sizeCheck = fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput); + if (sizeCheck!=MAGICNUMBER_SIZE) EXM_THROW(22, "Write error : cannot write header"); + + // Main Loop + while (1) + { + unsigned int outSize; + // Read Block + int inSize = (int) fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput); + if( inSize<=0 ) break; + filesize += inSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, inSize); + compressedfilesize += outSize+4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); + if (sizeCheck!=(size_t)(outSize+4)) EXM_THROW(23, "Write error : cannot write compressed block"); + } + + // Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +int compress_file_blockDependency(char* input_filename, char* output_filename, int compressionlevel) +{ + void* (*initFunction) (const char*); + int (*compressionFunction)(void*, const char*, char*, int, int); + char* (*translateFunction) (void*); + int (*freeFunction) (void*); + void* ctx; + unsigned long long filesize = 0; + unsigned long long compressedfilesize = 0; + unsigned int checkbits; + char* in_buff, *in_start, *in_end; + char* out_buff; + FILE* finput; + FILE* foutput; + clock_t start, end; + unsigned int blockSize, inputBufferSize; + size_t sizeCheck, header_size; + void* streamChecksumState=NULL; + + + // Init + start = clock(); + if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; + if (compressionlevel>=3) + { + initFunction = LZ4_createHC; + compressionFunction = LZ4_compressHC_limitedOutput_continue; + translateFunction = LZ4_slideInputBufferHC; + freeFunction = LZ4_freeHC; + } + else + { + initFunction = LZ4_create; + compressionFunction = LZ4_compress_limitedOutput_continue; + translateFunction = LZ4_slideInputBuffer; + freeFunction = LZ4_free; + } + get_fileHandle(input_filename, output_filename, &finput, &foutput); + blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); + + // Allocate Memory + inputBufferSize = blockSize + 64 KB; + if (inputBufferSize < MIN_STREAM_BUFSIZE) inputBufferSize = MIN_STREAM_BUFSIZE; + in_buff = (char*)malloc(inputBufferSize); + out_buff = (char*)malloc(blockSize+CACHELINE); + if (!in_buff || !out_buff) EXM_THROW(31, "Allocation error : not enough memory"); + in_start = in_buff; in_end = in_buff + inputBufferSize; + if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + ctx = initFunction(in_buff); + + // Write Archive Header + *(unsigned int*)out_buff = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention + *(out_buff+4) = (1 & _2BITS) << 6 ; // Version('01') + *(out_buff+4) |= (blockIndependence & _1BIT) << 5; + *(out_buff+4) |= (blockChecksum & _1BIT) << 4; + *(out_buff+4) |= (streamChecksum & _1BIT) << 2; + *(out_buff+5) = (char)((blockSizeId & _3BITS) << 4); + checkbits = XXH32((out_buff+4), 2, LZ4S_CHECKSUM_SEED); + checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); + *(out_buff+6) = (unsigned char) checkbits; + header_size = 7; + sizeCheck = fwrite(out_buff, 1, header_size, foutput); + if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); + compressedfilesize += header_size; + + // Main Loop + while (1) + { + unsigned int outSize; + unsigned int inSize; + // Read Block + if ((in_start+blockSize) > in_end) in_start = translateFunction(ctx); + inSize = (unsigned int) fread(in_start, (size_t)1, (size_t)blockSize, finput); + if( inSize==0 ) break; // No more input : end of compression + filesize += inSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + if (streamChecksum) XXH32_update(streamChecksumState, in_start, inSize); + + // Compress Block + outSize = compressionFunction(ctx, in_start, out_buff+4, inSize, inSize-1); + if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += inSize+4; + if (blockChecksum) compressedfilesize+=4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + if (outSize > 0) + { + int sizeToWrite; + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + if (blockChecksum) + { + unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); + } + sizeToWrite = 4 + outSize + (4*blockChecksum); + sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); + if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); + + } + else // Copy Original + { + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(inSize|0x80000000); // Add Uncompressed flag + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); + sizeCheck = fwrite(in_start, 1, inSize, foutput); + if (sizeCheck!=(size_t)(inSize)) EXM_THROW(35, "Write error : cannot write block"); + if (blockChecksum) + { + unsigned int checksum = XXH32(in_start, inSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); + } + } + in_start += inSize; + } + + // End of Stream mark + * (unsigned int*) out_buff = LZ4S_EOS; + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += 4; + if (streamChecksum) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); + compressedfilesize += 4; + } + + // Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + freeFunction(ctx); + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +int compress_file(char* input_filename, char* output_filename, int compressionlevel) +{ + int (*compressionFunction)(const char*, char*, int, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = 0; + unsigned int checkbits; + char* in_buff; + char* out_buff; + char* headerBuffer; + FILE* finput; + FILE* foutput; + clock_t start, end; + int blockSize; + size_t sizeCheck, header_size, readSize; + void* streamChecksumState=NULL; + + // Branch out + if (blockIndependence==0) return compress_file_blockDependency(input_filename, output_filename, compressionlevel); + + // Init + start = clock(); + if ((displayLevel==2) && (compressionlevel>=3)) displayLevel=3; + if (compressionlevel < 3) compressionFunction = LZ4_compress_limitedOutput; else compressionFunction = LZ4_compressHC_limitedOutput; + get_fileHandle(input_filename, output_filename, &finput, &foutput); + blockSize = LZ4S_GetBlockSize_FromBlockId (blockSizeId); + + // Allocate Memory + in_buff = (char*)malloc(blockSize); + out_buff = (char*)malloc(blockSize+CACHELINE); + headerBuffer = (char*)malloc(LZ4S_MAXHEADERSIZE); + if (!in_buff || !out_buff || !(headerBuffer)) EXM_THROW(31, "Allocation error : not enough memory"); + if (streamChecksum) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + + // Write Archive Header + *(unsigned int*)headerBuffer = LITTLE_ENDIAN_32(LZ4S_MAGICNUMBER); // Magic Number, in Little Endian convention + *(headerBuffer+4) = (1 & _2BITS) << 6 ; // Version('01') + *(headerBuffer+4) |= (blockIndependence & _1BIT) << 5; + *(headerBuffer+4) |= (blockChecksum & _1BIT) << 4; + *(headerBuffer+4) |= (streamChecksum & _1BIT) << 2; + *(headerBuffer+5) = (char)((blockSizeId & _3BITS) << 4); + checkbits = XXH32((headerBuffer+4), 2, LZ4S_CHECKSUM_SEED); + checkbits = LZ4S_GetCheckBits_FromXXH(checkbits); + *(headerBuffer+6) = (unsigned char) checkbits; + header_size = 7; + + // Write header + sizeCheck = fwrite(headerBuffer, 1, header_size, foutput); + if (sizeCheck!=header_size) EXM_THROW(32, "Write error : cannot write header"); + compressedfilesize += header_size; + + // read first block + readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + + // Main Loop + while (readSize>0) + { + unsigned int outSize; + + filesize += readSize; + DISPLAYLEVEL(3, "\rRead : %i MB ", (int)(filesize>>20)); + if (streamChecksum) XXH32_update(streamChecksumState, in_buff, (int)readSize); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, (int)readSize, (int)readSize-1); + if (outSize > 0) compressedfilesize += outSize+4; else compressedfilesize += readSize+4; + if (blockChecksum) compressedfilesize+=4; + DISPLAYLEVEL(3, "\rRead : %i MB ==> %.2f%% ", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + if (outSize > 0) + { + int sizeToWrite; + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(outSize); + if (blockChecksum) + { + unsigned int checksum = XXH32(out_buff+4, outSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) (out_buff+4+outSize) = LITTLE_ENDIAN_32(checksum); + } + sizeToWrite = 4 + outSize + (4*blockChecksum); + sizeCheck = fwrite(out_buff, 1, sizeToWrite, foutput); + if (sizeCheck!=(size_t)(sizeToWrite)) EXM_THROW(33, "Write error : cannot write compressed block"); + } + else // Copy Original Uncompressed + { + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(((unsigned long)readSize)|0x80000000); // Add Uncompressed flag + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(34, "Write error : cannot write block header"); + sizeCheck = fwrite(in_buff, 1, readSize, foutput); + if (sizeCheck!=readSize) EXM_THROW(35, "Write error : cannot write block"); + if (blockChecksum) + { + unsigned int checksum = XXH32(in_buff, (int)readSize, LZ4S_CHECKSUM_SEED); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(36, "Write error : cannot write block checksum"); + } + } + + // Read next block + readSize = fread(in_buff, (size_t)1, (size_t)blockSize, finput); + } + + // End of Stream mark + * (unsigned int*) out_buff = LZ4S_EOS; + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write end of stream"); + compressedfilesize += 4; + if (streamChecksum) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + * (unsigned int*) out_buff = LITTLE_ENDIAN_32(checksum); + sizeCheck = fwrite(out_buff, 1, 4, foutput); + if (sizeCheck!=(size_t)(4)) EXM_THROW(37, "Write error : cannot write stream checksum"); + compressedfilesize += 4; + } + + // Close & Free + free(in_buff); + free(out_buff); + free(headerBuffer); + fclose(finput); + fclose(foutput); + + // Final Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + return 0; +} + + +unsigned long long decodeLegacyStream(FILE* finput, FILE* foutput) +{ + unsigned long long filesize = 0; + char* in_buff; + char* out_buff; + unsigned int blockSize; + + + // Allocate Memory + in_buff = (char*)malloc(LZ4_compressBound(LEGACY_BLOCKSIZE)); + out_buff = (char*)malloc(LEGACY_BLOCKSIZE); + if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory"); + + // Main Loop + while (1) + { + int decodeSize; + size_t sizeCheck; + + // Block Size + sizeCheck = fread(&blockSize, 1, 4, finput); + if (sizeCheck==0) break; // Nothing to read : file read is completed + blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to Little Endian + if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) + { // Cannot read next block : maybe new stream ? + fseek(finput, -4, SEEK_CUR); + break; + } + + // Read Block + sizeCheck = fread(in_buff, 1, blockSize, finput); + + // Decode Block + decodeSize = LZ4_decompress_safe(in_buff, out_buff, blockSize, LEGACY_BLOCKSIZE); + if (decodeSize < 0) EXM_THROW(52, "Decoding Failed ! Corrupted input detected !"); + filesize += decodeSize; + + // Write Block + sizeCheck = fwrite(out_buff, 1, decodeSize, foutput); + if (sizeCheck != (size_t)decodeSize) EXM_THROW(53, "Write error : cannot write decoded block into output\n"); + } + + // Free + free(in_buff); + free(out_buff); + + return filesize; +} + + +unsigned long long decodeLZ4S(FILE* finput, FILE* foutput) +{ + unsigned long long filesize = 0; + char* in_buff; + char* out_buff, *out_start, *out_end; + unsigned char descriptor[LZ4S_MAXHEADERSIZE]; + size_t nbReadBytes; + int decodedBytes=0; + unsigned int maxBlockSize; + size_t sizeCheck; + int blockChecksumFlag, streamChecksumFlag, blockIndependenceFlag; + void* streamChecksumState=NULL; + int (*decompressionFunction)(const char*, char*, int, int) = LZ4_decompress_safe; + unsigned int prefix64k = 0; + + // Decode stream descriptor + nbReadBytes = fread(descriptor, 1, 3, finput); + if (nbReadBytes != 3) EXM_THROW(61, "Unreadable header"); + { + int version = (descriptor[0] >> 6) & _2BITS; + int streamSize = (descriptor[0] >> 3) & _1BIT; + int reserved1 = (descriptor[0] >> 1) & _1BIT; + int dictionary = (descriptor[0] >> 0) & _1BIT; + + int reserved2 = (descriptor[1] >> 7) & _1BIT; + int blockSizeId = (descriptor[1] >> 4) & _3BITS; + int reserved3 = (descriptor[1] >> 0) & _4BITS; + int checkBits = (descriptor[2] >> 0) & _8BITS; + int checkBits_xxh32; + + blockIndependenceFlag=(descriptor[0] >> 5) & _1BIT; + blockChecksumFlag = (descriptor[0] >> 4) & _1BIT; + streamChecksumFlag= (descriptor[0] >> 2) & _1BIT; + + if (version != 1) EXM_THROW(62, "Wrong version number"); + if (streamSize == 1) EXM_THROW(64, "Does not support stream size"); + if (reserved1 != 0) EXM_THROW(65, "Wrong value for reserved bits"); + if (dictionary == 1) EXM_THROW(66, "Does not support dictionary"); + if (reserved2 != 0) EXM_THROW(67, "Wrong value for reserved bits"); + if (blockSizeId < 4) EXM_THROW(68, "Unsupported block size"); + if (reserved3 != 0) EXM_THROW(67, "Wrong value for reserved bits"); + maxBlockSize = LZ4S_GetBlockSize_FromBlockId(blockSizeId); + // Checkbits verification + descriptor[1] &= 0xF0; + checkBits_xxh32 = XXH32(descriptor, 2, LZ4S_CHECKSUM_SEED); + checkBits_xxh32 = LZ4S_GetCheckBits_FromXXH(checkBits_xxh32); + if (checkBits != checkBits_xxh32) EXM_THROW(69, "Stream descriptor error detected"); + } + + if (!blockIndependenceFlag) + { + decompressionFunction = LZ4_decompress_safe_withPrefix64k; + prefix64k = 64 KB; + } + + // Allocate Memory + { + unsigned int outbuffSize = prefix64k+maxBlockSize; + in_buff = (char*)malloc(maxBlockSize); + if (outbuffSize < MIN_STREAM_BUFSIZE) outbuffSize = MIN_STREAM_BUFSIZE; + out_buff = (char*)malloc(outbuffSize); + out_end = out_buff + outbuffSize; + out_start = out_buff + prefix64k; + if (!in_buff || !out_buff) EXM_THROW(70, "Allocation error : not enough memory"); + } + if (streamChecksumFlag) streamChecksumState = XXH32_init(LZ4S_CHECKSUM_SEED); + + // Main Loop + while (1) + { + unsigned int blockSize, uncompressedFlag; + + // Block Size + nbReadBytes = fread(&blockSize, 1, 4, finput); + if( nbReadBytes != 4 ) EXM_THROW(71, "Read error : cannot read next block size"); + if (blockSize == LZ4S_EOS) break; // End of Stream Mark : stream is completed + blockSize = LITTLE_ENDIAN_32(blockSize); // Convert to little endian + uncompressedFlag = blockSize >> 31; + blockSize &= 0x7FFFFFFF; + if (blockSize > maxBlockSize) EXM_THROW(72, "Error : invalid block size"); + + // Read Block + nbReadBytes = fread(in_buff, 1, blockSize, finput); + if( nbReadBytes != blockSize ) EXM_THROW(73, "Read error : cannot read data block" ); + + // Check Block + if (blockChecksumFlag) + { + unsigned int checksum = XXH32(in_buff, blockSize, LZ4S_CHECKSUM_SEED); + unsigned int readChecksum; + sizeCheck = fread(&readChecksum, 1, 4, finput); + if( sizeCheck != 4 ) EXM_THROW(74, "Read error : cannot read next block size"); + readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian + if (checksum != readChecksum) EXM_THROW(75, "Error : invalid block checksum detected"); + } + + if (uncompressedFlag) + { + // Write uncompressed Block + sizeCheck = fwrite(in_buff, 1, blockSize, foutput); + if (sizeCheck != (size_t)blockSize) EXM_THROW(76, "Write error : cannot write data block"); + filesize += blockSize; + if (streamChecksumFlag) XXH32_update(streamChecksumState, in_buff, blockSize); + if (!blockIndependenceFlag) + { + if (blockSize >= prefix64k) + { + memcpy(out_buff, in_buff + (blockSize - prefix64k), prefix64k); // Required for reference for next blocks + out_start = out_buff + prefix64k; + continue; + } + else + { + memcpy(out_start, in_buff, blockSize); + decodedBytes = blockSize; + } + } + } + else + { + // Decode Block + decodedBytes = decompressionFunction(in_buff, out_start, blockSize, maxBlockSize); + if (decodedBytes < 0) EXM_THROW(77, "Decoding Failed ! Corrupted input detected !"); + filesize += decodedBytes; + if (streamChecksumFlag) XXH32_update(streamChecksumState, out_start, decodedBytes); + + // Write Block + sizeCheck = fwrite(out_start, 1, decodedBytes, foutput); + if (sizeCheck != (size_t)decodedBytes) EXM_THROW(78, "Write error : cannot write decoded block\n"); + } + + if (!blockIndependenceFlag) + { + out_start += decodedBytes; + if ((size_t)(out_end - out_start) < (size_t)maxBlockSize) + { + memcpy(out_buff, out_start - prefix64k, prefix64k); + out_start = out_buff + prefix64k; + } + } + } + + // Stream Checksum + if (streamChecksumFlag) + { + unsigned int checksum = XXH32_digest(streamChecksumState); + unsigned int readChecksum; + sizeCheck = fread(&readChecksum, 1, 4, finput); + if (sizeCheck != 4) EXM_THROW(74, "Read error : cannot read stream checksum"); + readChecksum = LITTLE_ENDIAN_32(readChecksum); // Convert to little endian + if (checksum != readChecksum) EXM_THROW(75, "Error : invalid stream checksum detected"); + } + + // Free + free(in_buff); + free(out_buff); + + return filesize; +} + + +unsigned long long selectDecoder( FILE* finput, FILE* foutput) +{ + unsigned int magicNumber, size; + int errorNb; + size_t nbReadBytes; + + // Check Archive Header + nbReadBytes = fread(&magicNumber, 1, MAGICNUMBER_SIZE, finput); + if (nbReadBytes==0) return 0; // EOF + if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(41, "Unrecognized header : Magic Number unreadable"); + magicNumber = LITTLE_ENDIAN_32(magicNumber); // Convert to Little Endian format + if (LZ4S_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4S_SKIPPABLE0; // fold skippable magic numbers + + switch(magicNumber) + { + case LZ4S_MAGICNUMBER: + return DEFAULT_DECOMPRESSOR(finput, foutput); + case LEGACY_MAGICNUMBER: + DISPLAYLEVEL(4, "Detected : Legacy format \n"); + return decodeLegacyStream(finput, foutput); + case LZ4S_SKIPPABLE0: + DISPLAYLEVEL(4, "Skipping detected skippable area \n"); + nbReadBytes = fread(&size, 1, 4, finput); + if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable"); + size = LITTLE_ENDIAN_32(size); // Convert to Little Endian format + errorNb = fseek(finput, size, SEEK_CUR); + if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area"); + return selectDecoder(finput, foutput); + EXTENDED_FORMAT; + default: + if (ftell(finput) == MAGICNUMBER_SIZE) EXM_THROW(44,"Unrecognized header : file cannot be decoded"); // Wrong magic number at the beginning of 1st stream + DISPLAYLEVEL(2, "Stream followed by unrecognized data\n"); + return 0; + } +} + + +int decodeFile(char* input_filename, char* output_filename) +{ + unsigned long long filesize = 0, decodedSize=0; + FILE* finput; + FILE* foutput; + clock_t start, end; + + + // Init + start = clock(); + get_fileHandle(input_filename, output_filename, &finput, &foutput); + + // Loop over multiple streams + do + { + decodedSize = selectDecoder(finput, foutput); + filesize += decodedSize; + } while (decodedSize); + + // Final Status + end = clock(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close + fclose(finput); + fclose(foutput); + + // Error status = OK + return 0; +} + + +void waitEnter() +{ + DISPLAY("Press enter to continue...\n"); + getchar(); +} + + +int main(int argc, char** argv) +{ + int i, + cLevel=0, + decode=0, + bench=0, + filenamesStart=2, + legacy_format=0, + forceStdout=0, + forceCompress=0, + pause=0; + char* input_filename=0; + char* output_filename=0; + char* dynNameSpace=0; + char nullOutput[] = NULL_OUTPUT; + char extension[] = LZ4_EXTENSION; + + // Init + programName = argv[0]; + + for(i=1; i='1') && (argument[1] <='9')) + { + int iters = argument[1] - '0'; + BMK_SetNbIterations(iters); + argument++; + } + break; + + // Pause at the end (hidden option) + case 'p': pause=1; BMK_SetPause(); break; + + EXTENDED_ARGUMENTS; + + // Unrecognised command + default : badusage(); + } + } + continue; + } + + // first provided filename is input + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + + // second provided filename is output + if (!output_filename) + { + output_filename=argument; + if (!strcmp (output_filename, nullOutput)) output_filename = nulmark; + continue; + } + } + + DISPLAYLEVEL(3, WELCOME_MESSAGE); + DISPLAYLEVEL(4, "Blocks size : %i KB\n", (1 << ((blockSizeId*2)-2))); + + // No input filename ==> use stdin + if(!input_filename) { input_filename=stdinmark; } + + // Check if input or output are defined as console; trigger an error in this case + if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); + + // Check if benchmark is selected + if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); + + // No output filename ==> try to select one automatically (when possible) + while (!output_filename) + { + if (!IS_CONSOLE(stdout)) { output_filename=stdoutmark; break; } // Default to stdout whenever possible (i.e. not a console) + if ((!decode) && !(forceCompress)) // auto-determine compression or decompression, based on file extension + { + size_t l = strlen(input_filename); + if (!strcmp(input_filename+(l-4), LZ4_EXTENSION)) decode=1; + } + if (!decode) // compression to file + { + size_t l = strlen(input_filename); + dynNameSpace = (char*)calloc(1,l+5); + output_filename = dynNameSpace; + strcpy(output_filename, input_filename); + strcpy(output_filename+l, LZ4_EXTENSION); + DISPLAYLEVEL(2, "Compressed filename will be : %s \n", output_filename); + break; + } + // decompression to file (automatic name will work only if input filename has correct format extension) + { + size_t outl; + size_t inl = strlen(input_filename); + dynNameSpace = (char*)calloc(1,inl+1); + output_filename = dynNameSpace; + strcpy(output_filename, input_filename); + outl = inl; + if (inl>4) + while ((outl >= inl-4) && (input_filename[outl] == extension[outl-inl+4])) output_filename[outl--]=0; + if (outl != inl-5) { DISPLAYLEVEL(1, "Cannot determine an output filename\n"); badusage(); } + DISPLAYLEVEL(2, "Decoding file %s \n", output_filename); + } + } + + // No warning message in pure pipe mode (stdin + stdout) + if (!strcmp(input_filename, stdinmark) && !strcmp(output_filename,stdoutmark) && (displayLevel==2)) displayLevel=1; + + // Check if input or output are defined as console; trigger an error in this case + if (!strcmp(input_filename, stdinmark) && IS_CONSOLE(stdin) ) badusage(); + if (!strcmp(output_filename,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) badusage(); + + // Decompress input if selected + if (decode) decodeFile(input_filename, output_filename); + else + // compression is default action + { + if (legacy_format) + { + DISPLAYLEVEL(3, "! Generating compressed LZ4 using Legacy format (deprecated !) ! \n"); + legacy_compress_file(input_filename, output_filename, cLevel); + } + else + { + DEFAULT_COMPRESSOR(input_filename, output_filename, cLevel); + } + } + + if (pause) waitEnter(); + free(dynNameSpace); + return 0; +} diff --git a/programs/xxhash.c b/programs/xxhash.c new file mode 100644 index 0000000..8304ec2 --- /dev/null +++ b/programs/xxhash.c @@ -0,0 +1,475 @@ +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : http://code.google.com/p/xxhash/ +*/ + + +//************************************** +// Tuning parameters +//************************************** +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. +// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +// XXH_ACCEPT_NULL_INPUT_POINTER : +// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. +// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. +// This option has a very small performance cost (only measurable on small inputs). +// By default, this option is disabled. To enable it, uncomment below define : +//#define XXH_ACCEPT_NULL_INPUT_POINTER 1 + +// XXH_FORCE_NATIVE_FORMAT : +// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. +// Results are therefore identical for little-endian and big-endian CPU. +// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. +// Should endian-independance be of no importance for your application, you may set the #define below to 1. +// It will improve speed for Big-endian CPU. +// This option has no impact on Little_Endian CPU. +#define XXH_FORCE_NATIVE_FORMAT 0 + + +//************************************** +// Compiler Specific Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +//************************************** +// Includes & Memory related functions +//************************************** +#include "xxhash.h" +// Modify the local functions below should you wish to use some other memory related routines +// for malloc(), free() +#include +FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } +FORCE_INLINE void XXH_free (void* p) { free(p); } +// for memcpy() +#include +FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U32_S { U32 v; } _PACKED U32_S; + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A32(x) (((U32_S *)(x))->v) + + +//*************************************** +// Compiler-specific Functions and Macros +//*************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + +#if defined(_MSC_VER) // Visual Studio +# define XXH_swap32 _byteswap_ulong +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static inline U32 XXH_swap32 (U32 x) { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff );} +#endif + + +//************************************** +// Constants +//************************************** +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + + +//************************************** +// Architecture Macros +//************************************** +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch + static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +#endif + + +//************************************** +// Macros +//************************************** +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations + + +//**************************** +// Memory reads +//**************************** +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + else + return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } + + +//**************************** +// Simple Hash Functions +//**************************** +FORCE_INLINE U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U32 h32; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p<=bEnd-4) + { + h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32(const void* input, int len, U32 seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + void* state = XXH32_init(seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +//**************************** +// Advanced Hash Functions +//**************************** + +struct XXH_state32_t +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + int memsize; + char memory[16]; +}; + + +int XXH32_sizeofState() +{ + XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough + return sizeof(struct XXH_state32_t); +} + + +XXH_errorcode XXH32_resetState(void* state_in, U32 seed) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +void* XXH32_init (U32 seed) +{ + void* state = XXH_malloc (sizeof(struct XXH_state32_t)); + XXH32_resetState(state, seed); + return state; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); + { + const U32* p32 = (const U32*)state->memory; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (void* state_in, const void* input, int len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + BYTE* bEnd = (BYTE*)state->memory + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p<=bEnd-4) + { + h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_intermediateDigest (void* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); + else + return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); +} + + +U32 XXH32_digest (void* state_in) +{ + U32 h32 = XXH32_intermediateDigest(state_in); + + XXH_free(state_in); + + return h32; +} diff --git a/programs/xxhash.h b/programs/xxhash.h new file mode 100644 index 0000000..a319bcc --- /dev/null +++ b/programs/xxhash.h @@ -0,0 +1,164 @@ +/* + xxHash - Fast Hash algorithm + Header File + Copyright (C) 2012-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : http://code.google.com/p/xxhash/ +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +//**************************** +// Type +//**************************** +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + + +//**************************** +// Simple Hash Functions +//**************************** + +unsigned int XXH32 (const void* input, int len, unsigned int seed); + +/* +XXH32() : + Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". + The memory between input & input+len must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s + Note that "len" is type "int", which means it is limited to 2^31-1. + If your data is larger, use the advanced functions below. +*/ + + + +//**************************** +// Advanced Hash Functions +//**************************** + +void* XXH32_init (unsigned int seed); +XXH_errorcode XXH32_update (void* state, const void* input, int len); +unsigned int XXH32_digest (void* state); + +/* +These functions calculate the xxhash of an input provided in several small packets, +as opposed to an input provided as a single block. + +It must be started with : +void* XXH32_init() +The function returns a pointer which holds the state of calculation. + +This pointer must be provided as "void* state" parameter for XXH32_update(). +XXH32_update() can be called as many times as necessary. +The user must provide a valid (allocated) input. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. +Note that "len" is type "int", which means it is limited to 2^31-1. +If your data is larger, it is recommended to chunk your data into blocks +of size for example 2^30 (1GB) to avoid any "int" overflow issue. + +Finally, you can end the calculation anytime, by using XXH32_digest(). +This function returns the final 32-bits hash. +You must provide the same "void* state" parameter created by XXH32_init(). +Memory will be freed by XXH32_digest(). +*/ + + +int XXH32_sizeofState(); +XXH_errorcode XXH32_resetState(void* state, unsigned int seed); + +#define XXH32_SIZEOFSTATE 48 +typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; +/* +These functions allow user application to make its own allocation for state. + +XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state. +Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. +This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state. + +For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), +use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. +*/ + + +unsigned int XXH32_intermediateDigest (void* state); +/* +This function does the same as XXH32_digest(), generating a 32-bit hash, +but preserve memory context. +This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update(). +To free memory context, use XXH32_digest(), or free(). +*/ + + + +//**************************** +// Deprecated function names +//**************************** +// The following translations are provided to ease code transition +// You are encouraged to no longer this function names +#define XXH32_feed XXH32_update +#define XXH32_result XXH32_digest +#define XXH32_getIntermediateResult XXH32_intermediateDigest + + + +#if defined (__cplusplus) +} +#endif diff --git a/xxhash.c b/xxhash.c deleted file mode 100644 index 8304ec2..0000000 --- a/xxhash.c +++ /dev/null @@ -1,475 +0,0 @@ -/* -xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -*/ - - -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_USE_UNALIGNED_ACCESS 1 -#endif - -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -//#define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. -#define XXH_FORCE_NATIVE_FORMAT 0 - - -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#ifdef _MSC_VER // Visual Studio -# define FORCE_INLINE static __forceinline -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -//************************************** -// Includes & Memory related functions -//************************************** -#include "xxhash.h" -// Modify the local functions below should you wish to use some other memory related routines -// for malloc(), free() -#include -FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } -FORCE_INLINE void XXH_free (void* p) { free(p); } -// for memcpy() -#include -FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S { U32 v; } _PACKED U32_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A32(x) (((U32_S *)(x))->v) - - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif - -#if defined(_MSC_VER) // Visual Studio -# define XXH_swap32 _byteswap_ulong -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static inline U32 XXH_swap32 (U32 x) { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff );} -#endif - - -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - - -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch - static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) -#endif - - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } - - -//**************************** -// Simple Hash Functions -//**************************** -FORCE_INLINE U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U32 h32; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; } -#endif - - if (len>=16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do - { - v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } - else - { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32(const void* input, int len, U32 seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - void* state = XXH32_init(seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -//**************************** -// Advanced Hash Functions -//**************************** - -struct XXH_state32_t -{ - U64 total_len; - U32 seed; - U32 v1; - U32 v2; - U32 v3; - U32 v4; - int memsize; - char memory[16]; -}; - - -int XXH32_sizeofState() -{ - XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough - return sizeof(struct XXH_state32_t); -} - - -XXH_errorcode XXH32_resetState(void* state_in, U32 seed) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - - -void* XXH32_init (U32 seed) -{ - void* state = XXH_malloc (sizeof(struct XXH_state32_t)); - XXH32_resetState(state, seed); - return state; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) // fill in tmp buffer - { - XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); - { - const U32* p32 = (const U32*)state->memory; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do - { - v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->memory, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH32_update (void* state_in, const void* input, int len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE * p = (const BYTE*)state->memory; - BYTE* bEnd = (BYTE*)state->memory + state->memsize; - U32 h32; - - if (state->total_len >= 16) - { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32) state->total_len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32_intermediateDigest (void* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); - else - return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); -} - - -U32 XXH32_digest (void* state_in) -{ - U32 h32 = XXH32_intermediateDigest(state_in); - - XXH_free(state_in); - - return h32; -} diff --git a/xxhash.h b/xxhash.h deleted file mode 100644 index a319bcc..0000000 --- a/xxhash.h +++ /dev/null @@ -1,164 +0,0 @@ -/* - xxHash - Fast Hash algorithm - Header File - Copyright (C) 2012-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. -*/ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -//**************************** -// Type -//**************************** -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - - -//**************************** -// Simple Hash Functions -//**************************** - -unsigned int XXH32 (const void* input, int len, unsigned int seed); - -/* -XXH32() : - Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". - The memory between input & input+len must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - This function successfully passes all SMHasher tests. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s - Note that "len" is type "int", which means it is limited to 2^31-1. - If your data is larger, use the advanced functions below. -*/ - - - -//**************************** -// Advanced Hash Functions -//**************************** - -void* XXH32_init (unsigned int seed); -XXH_errorcode XXH32_update (void* state, const void* input, int len); -unsigned int XXH32_digest (void* state); - -/* -These functions calculate the xxhash of an input provided in several small packets, -as opposed to an input provided as a single block. - -It must be started with : -void* XXH32_init() -The function returns a pointer which holds the state of calculation. - -This pointer must be provided as "void* state" parameter for XXH32_update(). -XXH32_update() can be called as many times as necessary. -The user must provide a valid (allocated) input. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -Note that "len" is type "int", which means it is limited to 2^31-1. -If your data is larger, it is recommended to chunk your data into blocks -of size for example 2^30 (1GB) to avoid any "int" overflow issue. - -Finally, you can end the calculation anytime, by using XXH32_digest(). -This function returns the final 32-bits hash. -You must provide the same "void* state" parameter created by XXH32_init(). -Memory will be freed by XXH32_digest(). -*/ - - -int XXH32_sizeofState(); -XXH_errorcode XXH32_resetState(void* state, unsigned int seed); - -#define XXH32_SIZEOFSTATE 48 -typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; -/* -These functions allow user application to make its own allocation for state. - -XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state. -Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. -This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state. - -For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), -use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. -*/ - - -unsigned int XXH32_intermediateDigest (void* state); -/* -This function does the same as XXH32_digest(), generating a 32-bit hash, -but preserve memory context. -This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update(). -To free memory context, use XXH32_digest(), or free(). -*/ - - - -//**************************** -// Deprecated function names -//**************************** -// The following translations are provided to ease code transition -// You are encouraged to no longer this function names -#define XXH32_feed XXH32_update -#define XXH32_result XXH32_digest -#define XXH32_getIntermediateResult XXH32_intermediateDigest - - - -#if defined (__cplusplus) -} -#endif -- cgit v0.12