diff options
author | The Android Open Source Project <initial-contribution@android.com> | 2009-01-20 14:03:58 -0800 |
---|---|---|
committer | The Android Open Source Project <initial-contribution@android.com> | 2009-01-20 14:03:58 -0800 |
commit | 280cf4827c37fe3dd0f038536d28833154517e92 (patch) | |
tree | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | |
parent | 990c1ebf2f02d2ce5086983002ca466012088f6d (diff) | |
download | xdelta3-280cf4827c37fe3dd0f038536d28833154517e92.tar.gz |
auto import from //branches/cupcake/...@127101android-sdk-tools_r2android-sdk-1.6_r2android-sdk-1.6_r1android-sdk-1.6-docs_r1android-sdk-1.5_r3android-sdk-1.5_r1android-sdk-1.5-preandroid-1.6_r2android-1.6_r1.5android-1.6_r1.4android-1.6_r1.3android-1.6_r1.2android-1.6_r1.1android-1.6_r1android-1.5r4android-1.5r3android-1.5r2android-1.5donut-release2donut-releasedonutcupcake-releasecupcake
49 files changed, 0 insertions, 29892 deletions
diff --git a/Android.mk b/Android.mk deleted file mode 100644 index a6fa250..0000000 --- a/Android.mk +++ /dev/null @@ -1,35 +0,0 @@ -LOCAL_PATH:= $(call my-dir) - -xdelta3_cflags := \ - -O3 \ - -fno-function-sections -fno-data-sections -fno-inline \ - -DSUPPORT_ANDROID_PRELINK_TAGS \ - -DGENERIC_ENCODE_TABLES=0 \ - -DREGRESSION_TEST=0 \ - -DSECONDARY_DJW=1 \ - -DSECONDARY_FGK=1 \ - -DXD3_DEBUG=0 \ - -DXD3_MAIN=0 \ - -DXD3_POSIX=1 \ - -DXD3_USE_LARGEFILE64=1 - -include $(CLEAR_VARS) - -LOCAL_LDLIBS += -lm -LOCAL_CFLAGS += $(xdelta3_cflags) -LOCAL_SRC_FILES := xdelta3.c -LOCAL_C_INCLUDES:= $(LOCAL_PATH)/ -LOCAL_MODULE := libxdelta3 -include $(BUILD_STATIC_LIBRARY) - -include $(CLEAR_VARS) - -LOCAL_LDLIBS += -lm -LOCAL_CFLAGS += $(xdelta3_cflags) -DXD3_MAIN=1 -LOCAL_SRC_FILES := xdelta3.c -LOCAL_C_INCLUDES:= $(LOCAL_PATH)/ -LOCAL_MODULE := xdelta3 - -include $(BUILD_HOST_EXECUTABLE) - - diff --git a/COPYING b/COPYING deleted file mode 100644 index 5b6e7c6..0000000 --- a/COPYING +++ /dev/null @@ -1,340 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - <signature of Ty Coon>, 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. diff --git a/Makefile b/Makefile deleted file mode 100644 index bbc6d2a..0000000 --- a/Makefile +++ /dev/null @@ -1,310 +0,0 @@ -# xdelta 3 - delta compression tools and library -# Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - -UNAME = $(shell uname) -CYGWIN = $(findstring CYGWIN, $(UNAME)) -DARWIN = $(findstring Darwin, $(UNAME)) -PYVER = 2.5 - -ifeq ("$(CYGWIN)", "") -SWIGTGT = xdelta3module.so -PYTGT = build/lib.linux-i686-$(PYVER)/xdelta3main.so -else -SWIGTGT = xdelta3module.dll -PYTGT = build/lib.cygwin-1.5.24-i686-$(PYVER)/xdelta3main.dll -endif - -SOURCES = xdelta3-cfgs.h \ - xdelta3-decode.h \ - xdelta3-djw.h \ - xdelta3-fgk.h \ - xdelta3-hash.h \ - xdelta3-list.h \ - xdelta3-main.h \ - xdelta3-merge.h \ - xdelta3-python.h \ - xdelta3-second.h \ - xdelta3-test.h \ - xdelta3.c \ - xdelta3.h - -TARGETS = xdelta3-debug \ - xdelta3 \ - xdelta3-debug2 \ - xdelta3-debug3 \ - xdelta3.o \ - xdelta3_wrap.o \ - xdelta3-32 \ - xdelta3-64 \ - xdelta3-everything \ - xdelta3-Opg \ - xdelta3-64-O \ - xdelta3-Op \ - xdelta3-decoder xdelta3-decoder-nomain.o \ - xdelta3-nosec.o xdelta3-all.o xdelta3-fgk.o \ - xdelta3-noext xdelta3-tools \ - xdelta3-notools \ - xdelta3_wrap.c xdelta3.py \ - $(PYTGT) $(SWIGTGT) - -PYTHON = python - -WIXDIR = "/cygdrive/c/Program Files/wix2.0.4820" - -# -arch x86_64 -CFLAGS= -Wall -Wshadow -fno-builtin - -# $Format: "REL=$Xdelta3Version$" $ -REL=3.0u - -RELDIR = xdelta$(REL) - -EXTRA = Makefile COPYING linkxd3lib.c badcopy.c xdelta3.swig \ - draft-korn-vcdiff.txt xdelta3.vcproj badcopy.vcproj \ - xdelta3-regtest.py xdelta3-test.py setup.py \ - examples/Makefile examples/small_page_test.c \ - examples/README examples/encode_decode_test.c \ - examples/compare_test.c examples/speed_test.c \ - examples/test.h examples/checksum_test.cc \ - xdelta3.py xdelta3_wrap.c xdelta3.wxs xdelta3.wxi \ - testing/cmp.h testing/delta.h testing/file.h \ - testing/modify.h testing/random.h testing/segment.h \ - testing/sizes.h testing/test.h testing/Makefile \ - README readme.txt - -SWIG_FLAGS = -DXD3_DEBUG=1 \ - -DEXTERNAL_COMPRESSION=0 \ - -DXD3_USE_LARGEFILE64=1 \ - -DGENERIC_ENCODE_TABLES=1 \ - -DSECONDARY_DJW=1 \ - -DVCDIFF_TOOLS=1 \ - -DSWIG_MODULE=1 - -all: xdelta3-debug xdelta3 - -all-py: all $(PYTGT) $(SWIGTGT) - -all-targets: $(TARGETS) - -all-targets-test: all-targets test - -pytgt: $(PYTGT) -swigtgt: $(SWIGTGT) - -test: - ./xdelta3-debug test - -tar: - tar --exclude ".svn" -czf /tmp/$(RELDIR)-tmp.tar.gz $(SOURCES) $(EXTRA) - rm -rf /tmp/$(RELDIR) - mkdir /tmp/$(RELDIR) - (cd /tmp/$(RELDIR) && tar -xzf ../$(RELDIR)-tmp.tar.gz) - tar -czf ./$(RELDIR).tar.gz -C /tmp $(RELDIR) - +tar -tzf ./$(RELDIR).tar.gz - rm -rf /tmp/$(RELDIR) - -zip: - tar --exclude ".svn" -czf /tmp/$(RELDIR)-tmp.tar.gz $(SOURCES) $(EXTRA) - rm -rf /tmp/$(RELDIR) - mkdir /tmp/$(RELDIR) - (cd /tmp/$(RELDIR) && tar -xzf ../$(RELDIR)-tmp.tar.gz) - tar -czf ./$(RELDIR).tar.gz -C /tmp $(RELDIR) - +zip -r $(RELDIR).zip /tmp/$(RELDIR) - rm -rf /tmp/$(RELDIR) - -clean: - rm -f $(TARGETS) - rm -rf build Debug Release core cifs* *.stackdump *.exe \ - xdelta3.ncb xdelta3.suo xdelta3.sln xdelta3.wixobj xdelta3.msi - -wix: xdelta3.wxs xdelta3.wxi readme.txt Release\xdelta3.exe - $(WIXDIR)/candle.exe xdelta3.wxs -out xdelta3.wixobj - $(WIXDIR)/light.exe xdelta3.wixobj -out xdelta3.msi - -xdelta3: $(SOURCES) - $(CC) $(CFLAGS) -O3 xdelta3.c -lm -o xdelta3 \ - -DGENERIC_ENCODE_TABLES=0 \ - -DREGRESSION_TEST=1 \ - -DSECONDARY_DJW=1 \ - -DSECONDARY_FGK=1 \ - -DXD3_DEBUG=0 \ - -DXD3_MAIN=1 \ - -DXD3_POSIX=1 \ - -DXD3_USE_LARGEFILE64=1 - -xdelta3-debug: $(SOURCES) - $(CC) -g $(CFLAGS) xdelta3.c -lm -o xdelta3-debug \ - -DGENERIC_ENCODE_TABLES=1 \ - -DREGRESSION_TEST=1 \ - -DSECONDARY_DJW=1 \ - -DSECONDARY_FGK=1 \ - -DXD3_DEBUG=1 \ - -DXD3_MAIN=1 \ - -DXD3_STDIO=1 \ - -DXD3_USE_LARGEFILE64=1 - -xdelta3-32: $(SOURCES) - $(CC) -g $(CFLAGS) xdelta3.c -lm -o xdelta3-32 \ - -DXD3_DEBUG=1 \ - -DXD3_USE_LARGEFILE64=0 \ - -DREGRESSION_TEST=1 \ - -DSECONDARY_DJW=1 \ - -DSECONDARY_FGK=1 \ - -DXD3_MAIN=1 \ - -DXD3_POSIX=1 - -xdelta3-debug2: $(SOURCES) - $(CC) -g $(CFLAGS) \ - xdelta3.c -o xdelta3-debug2 \ - -DXD3_DEBUG=2 \ - -DXD3_MAIN=1 \ - -DXD3_STDIO=1 \ - -DXD3_USE_LARGEFILE64=1 \ - -DGENERIC_ENCODE_TABLES=1 \ - -DREGRESSION_TEST=1 \ - -DSECONDARY_DJW=1 \ - -DSECONDARY_FGK=1 \ - -lm - -xdelta3-debug3: $(SOURCES) - $(CC) -g $(CFLAGS) xdelta3.c -o xdelta3-debug3 \ - -DXD3_MAIN=1 \ - -DGENERIC_ENCODE_TABLES=1 \ - -DXD3_USE_LARGEFILE64=1 \ - -DXD3_STDIO=1 \ - -DREGRESSION_TEST=1 \ - -DXD3_DEBUG=3 \ - -DSECONDARY_DJW=1 \ - -DSECONDARY_FGK=1 \ - -lm - -$(PYTGT): $(SOURCES) setup.py - $(PYTHON) setup.py install --verbose --compile --force - -xdelta3_wrap.c xdelta3.py: xdelta3.swig - swig -python xdelta3.swig - -xdelta3.o: $(SOURCES) - $(CC) -O3 $(CFLAGS) -c xdelta3.c $(SWIG_FLAGS) -o xdelta3.o - -xdelta3_wrap.o: xdelta3_wrap.c - $(CC) -O3 $(CFLAGS) $(SWIG_FLAGS) \ - -DHAVE_CONFIG_H \ - -I/usr/include/python$(PYVER) \ - -I/usr/lib/python$(PYVER)/config \ - -fpic \ - -c xdelta3_wrap.c - -xdelta3module.dll: xdelta3_wrap.o xdelta3.o - gcc -shared -Wl,--enable-auto-image-base \ - xdelta3.o \ - xdelta3_wrap.o \ - -L/usr/lib/python$(PYVER)/config \ - -lpython$(PYVER) \ - -o xdelta3module.dll - cp $(SWIGTGT) /usr/lib/python$(PYVER)/site-packages - -ifeq ("$(DARWIN)", "") -xdelta3module.so: xdelta3_wrap.o xdelta3.o - ld -shared xdelta3.o xdelta3_wrap.o \ - -o xdelta3module.so \ - /usr/lib/libpython$(PYVER).so \ - -lc -else -xdelta3module.so: xdelta3_wrap.o xdelta3.o - gcc -Wl,-F. -bundle -undefined dynamic_lookup $(CFLAGS) \ - xdelta3.o xdelta3_wrap.o -o xdelta3module.so -endif - -xdelta3-decoder: $(SOURCES) - $(CC) -O3 -Wall -Wshadow xdelta3.c \ - -DXD3_ENCODER=0 -DXD3_MAIN=1 -DSECONDARY_FGK=0 -DSECONDARY_DJW=0 \ - -DXD3_STDIO=1 -DEXTERNAL_COMPRESSION=0 -DVCDIFF_TOOLS=0 \ - -o xdelta3-decoder - -xdelta3-decoder-nomain.o: $(SOURCES) linkxd3lib.c - $(CC) -O3 -Wall -Wshadow xdelta3.c linkxd3lib.c \ - -DXD3_ENCODER=0 -DSECONDARY_FGK=0 -DSECONDARY_DJW=0 \ - -o xdelta3-decoder-nomain.o - strip xdelta3-decoder-nomain.o - -xdelta3-O++: $(SOURCES) - $(CXX) -g -O3 $(CFLAGS) xdelta3.c \ - -o xdelta3-O++ \ - -DXD3_MAIN=1 \ - -DSECONDARY_DJW=1 \ - -DREGRESSION_TEST=1 \ - -lm - -xdelta3-Op: $(SOURCES) - $(CC) -g -O3 $(CFLAGS) xdelta3.c \ - -o xdelta3-Op \ - -DXD3_POSIX=1 \ - -DXD3_MAIN=1 \ - -DREGRESSION_TEST=1 \ - -lm - -xdelta3-64: $(SOURCES) - $(CC) -g $(CFLAGS) \ - xdelta3.c \ - -o xdelta3-64 \ - -DXD3_POSIX=1 \ - -DXD3_MAIN=1 \ - -DREGRESSION_TEST=1 \ - -DXD3_DEBUG=0 \ - -DXD3_USE_LARGEFILE64=1 \ - -lm - -xdelta3-64-O: $(SOURCES) - $(CC) -O3 $(CFLAGS) \ - xdelta3.c \ - -o xdelta3-64-O \ - -DXD3_POSIX=1 \ - -DXD3_MAIN=1 \ - -DXD3_USE_LARGEFILE64=1 \ - -lm - -xdelta3-everything: $(SOURCES) - $(CC) -g $(CFLAGS) \ - xdelta3.c \ - -o xdelta3-everything \ - -DXD3_MAIN=1 \ - -DVCDIFF_TOOLS=1 \ - -DREGRESSION_TEST=1 \ - -DSECONDARY_FGK=1 \ - -DSECONDARY_DJW=1 \ - -DGENERIC_ENCODE_TABLES=1 \ - -DGENERIC_ENCODE_TABLES_COMPUTE=1 \ - -DXD3_POSIX=1 \ - -DEXTERNAL_COMPRESSION=1 \ - -DXD3_DEBUG=1 \ - -lm - -xdelta3-Opg: $(SOURCES) - $(CC) -pg -g -O3 $(CFLAGS) \ - xdelta3.c \ - -o xdelta3-Opg \ - -DXD3_MAIN=1 \ - -DSECONDARY_DJW=1 \ - -DSECONDARY_FGK=1 \ - -DXD3_POSIX=1 \ - -DXD3_USE_LARGEFILE64=1 \ - -DREGRESSION_TEST=1 - -xdelta3-nosec.o: $(SOURCES) - $(CC) -O3 $(CFLAGS) -c xdelta3.c -DSECONDARY_FGK=0 -DSECONDARY_DJW=0 -o xdelta3-nosec.o - -xdelta3-all.o: $(SOURCES) - $(CC) -O3 $(CFLAGS) -c xdelta3.c -DSECONDARY_FGK=1 -DSECONDARY_DJW=1 -o xdelta3-all.o - -xdelta3-fgk.o: $(SOURCES) - $(CC) -O3 $(CFLAGS) -c xdelta3.c -DSECONDARY_FGK=1 -DSECONDARY_DJW=0 -o xdelta3-fgk.o - -xdelta3-noext: $(SOURCES) - $(CC) -O3 $(CFLAGS) xdelta3.c -DXD3_MAIN=1 -DEXTERNAL_COMPRESSION=0 -o xdelta3-noext - -xdelta3-tools: $(SOURCES) - $(CC) -O3 $(CFLAGS) xdelta3.c -DXD3_MAIN=1 -o xdelta3-tools - -xdelta3-notools: $(SOURCES) - $(CC) -O3 $(CFLAGS) xdelta3.c -DXD3_MAIN=1 -DVCDIFF_TOOLS=0 -o xdelta3-notools @@ -1,34 +0,0 @@ -Xdelta 3.x readme.txt -Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007 -<josh.macdonald@gmail.com> - - -Thanks for downloading Xdelta! - -This directory contains the Xdelta3 command-line interface (CLI) and source -distribution for VCDIFF differential compression, a.k.a. delta -compression. The latest information and downloads are available here: - - http://xdelta.org/ - http://code.google.com/p/xdelta/ - -The command-line syntax: - - http://code.google.com/p/xdelta/wiki/CommandLineSyntax - -Run 'xdelta3 -h' for brief help. Run 'xdelta3 test' for built-in tests. - -Sample commands (like gzip, -e means encode, -d means decode) - - xdelta3 -9 -S djw -e -vfs OLD_FILE NEW_FILE DELTA_FILE - xdelta3 -d -vfs OLD_FILE DELTA_FILE DECODED_FILE - -File bug reports and browse open support issues here: - - http://code.google.com/p/xdelta/issues/list - -The source distribution contains the C/C++/Python APIs, Unix, Microsoft VC++ -and Cygwin builds. Xdelta3 is covered under the terms of the GPL, see -COPYING. - -Commercial inquiries welcome, please contact <josh.macdonald@gmail.com> diff --git a/README.android b/README.android deleted file mode 100644 index bbe0b31..0000000 --- a/README.android +++ /dev/null @@ -1,7 +0,0 @@ -The contents of this directory are the xdelta3.0u package, downloaded from - - http://xdelta.googlecode.com/files/xdelta3.0u.tar.gz - -on 12 Oct 2008. I added the Android.mk file and this README.android -file to the directory; nothing else has been touched. - diff --git a/badcopy.c b/badcopy.c deleted file mode 100644 index 03abc63..0000000 --- a/badcopy.c +++ /dev/null @@ -1,158 +0,0 @@ -#include <stdio.h> -#include <stdlib.h> -#include <math.h> - -#define BUFSZ (1 << 22) - -#ifdef WIN32 -// whatever -static -double drand48() { - double r = rand() / (double)RAND_MAX; - return r; -} -long lrand48() { - long l = 0; - int i; - for (i = 0; i < 32; i++) { - l = l ^ (l << 2) ^ (l << 1) ^ rand(); - } - return l; -} -#endif - -#ifdef _WIN32 -#define XD3_WIN32 1 -#else -#define XD3_POSIX 1 -#endif -#define XD3_MAIN 1 -#define main notmain -#define EXTERNAL_COMPRESSION 0 -#define XD3_USE_LARGEFILE64 1 -#include "xdelta3.c" -#undef main - - -double error_prob = 0.0001; -usize_t mean_change = 100; -xoff_t total_change = 0; -xoff_t total_size = 0; -usize_t max_change = 0; -usize_t num_change = 0; - - -static usize_t -edist (usize_t mean, usize_t max) -{ - double mean_d = mean; - double erand = log (1.0 / drand48 ()); - usize_t x = (usize_t) (mean_d * erand + 0.5); - - return (x < max) ? (x > 0 ? x : 1) : max; -} - -void modify (char *buf, usize_t size) -{ - usize_t bufpos = 0, j; - usize_t last_end = 0; - - for (;; /* bufpos and j are incremented in the inner loop */) - { - /* The size of the next modification. */ - usize_t next_size = edist (mean_change, 1 << 31); - /* The expected interval of such a change. */ - double expect_interval = ((double) next_size * (1.0 - error_prob)) / error_prob; - /* The number of bytes until the next modification. */ - usize_t next_mod = edist ((usize_t)expect_interval, 1 << 31); - - if (next_size + next_mod + bufpos > size) { break; } - - if (max_change < next_size) { max_change = next_size; } - - bufpos += next_mod; - - fprintf (stderr, "COPY: %I64u-%I64u (%u)\n", - total_size + (xoff_t)last_end, - total_size + (xoff_t)bufpos, - bufpos - last_end); - fprintf (stderr, "ADD: %I64u-%I64u (%u) is change %u\n", - total_size + (xoff_t)bufpos, - total_size + (xoff_t)(bufpos + next_size), - next_size, num_change); - - total_change += next_size; - num_change += 1; - - for (j = 0; j < next_size; j += 1, bufpos += 1) - { - buf[bufpos] = (char)(lrand48 () >> 3); - } - - last_end = bufpos; - } - - fprintf (stderr, "COPY: %I64u-%I64u (%u)\n", - total_size + last_end, - total_size + size, size - last_end); - - total_size += size; -} - -int main(int argc, char **argv) -{ - main_file inp, out; - char *buf = malloc(BUFSZ); - int c, ret; - main_file_init(&inp); - main_file_init(&out); - option_force = 1; - if (argc > 5) - { - fprintf (stderr, "usage: badcopy [byte_error_prob [mean_error_size]]\n"); - return 1; - } - - if (argc > 4) { mean_change = atoi (argv[4]); } - if (argc > 3) { error_prob = atof (argv[3]); } - fprintf (stderr, "mean change = %u; error_prob = %0.10f\n", mean_change, error_prob); - - if ((ret = main_file_open (&inp, argv[1], XO_READ)) != 0) { - return 1; - } - if ((ret = main_file_open (&out, argv[2], XO_WRITE)) != 0) { - return 1; - } - - if (error_prob < 0.0 || error_prob > 1.0) - { - fprintf (stderr, "warning: error probability out of range\n"); - return 1; - } - - do - { - if ((ret = main_file_read (&inp, buf, BUFSZ, &c, "read failed")) != 0) { - return 1; - } - - if (c == 0) { break; } - - modify (buf, c); - - if ((ret = main_file_write (&out, buf, c, "write failed")) != 0) { - return 1; - } - } - while (c == BUFSZ); - - if ((ret = main_file_close (&out))) - { - return 1; - } - - fprintf (stderr, "add_prob %f; %u adds; total_change %u of %u bytes; add percentage %f; max add size %u\n", - error_prob, num_change, total_change, total_size, (double) total_change / (double) total_size, max_change); - - return 0; -} diff --git a/badcopy.vcproj b/badcopy.vcproj deleted file mode 100644 index 50683f6..0000000 --- a/badcopy.vcproj +++ /dev/null @@ -1,218 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?> -<VisualStudioProject - ProjectType="Visual C++" - Version="8.00" - Name="badcopy" - ProjectGUID="{FED2964C-7114-41AC-81EE-68A4D2B67503}" - RootNamespace="badcopy" - Keyword="Win32Proj" - > - <Platforms> - <Platform - Name="Win32" - /> - </Platforms> - <ToolFiles> - </ToolFiles> - <Configurations> - <Configuration - Name="Debug|Win32" - OutputDirectory="Debug" - IntermediateDirectory="Debug" - ConfigurationType="1" - > - <Tool - Name="VCPreBuildEventTool" - /> - <Tool - Name="VCCustomBuildTool" - /> - <Tool - Name="VCXMLDataGeneratorTool" - /> - <Tool - Name="VCWebServiceProxyGeneratorTool" - /> - <Tool - Name="VCMIDLTool" - /> - <Tool - Name="VCCLCompilerTool" - Optimization="0" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;" - MinimalRebuild="true" - BasicRuntimeChecks="3" - RuntimeLibrary="3" - UsePrecompiledHeader="0" - WarningLevel="3" - Detect64BitPortabilityProblems="true" - DebugInformationFormat="4" - /> - <Tool - Name="VCManagedResourceCompilerTool" - /> - <Tool - Name="VCResourceCompilerTool" - /> - <Tool - Name="VCPreLinkEventTool" - /> - <Tool - Name="VCLinkerTool" - LinkIncremental="2" - GenerateDebugInformation="true" - SubSystem="1" - TargetMachine="1" - /> - <Tool - Name="VCALinkTool" - /> - <Tool - Name="VCManifestTool" - /> - <Tool - Name="VCXDCMakeTool" - /> - <Tool - Name="VCBscMakeTool" - /> - <Tool - Name="VCFxCopTool" - /> - <Tool - Name="VCAppVerifierTool" - /> - <Tool - Name="VCWebDeploymentTool" - /> - <Tool - Name="VCPostBuildEventTool" - /> - </Configuration> - <Configuration - Name="Release|Win32" - OutputDirectory="Release" - IntermediateDirectory="Release" - ConfigurationType="1" - > - <Tool - Name="VCPreBuildEventTool" - /> - <Tool - Name="VCCustomBuildTool" - /> - <Tool - Name="VCXMLDataGeneratorTool" - /> - <Tool - Name="VCWebServiceProxyGeneratorTool" - /> - <Tool - Name="VCMIDLTool" - /> - <Tool - Name="VCCLCompilerTool" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;" - RuntimeLibrary="2" - UsePrecompiledHeader="0" - WarningLevel="3" - Detect64BitPortabilityProblems="true" - DebugInformationFormat="3" - /> - <Tool - Name="VCManagedResourceCompilerTool" - /> - <Tool - Name="VCResourceCompilerTool" - /> - <Tool - Name="VCPreLinkEventTool" - /> - <Tool - Name="VCLinkerTool" - LinkIncremental="2" - GenerateDebugInformation="true" - SubSystem="1" - OptimizeReferences="2" - EnableCOMDATFolding="2" - TargetMachine="1" - /> - <Tool - Name="VCALinkTool" - /> - <Tool - Name="VCManifestTool" - /> - <Tool - Name="VCXDCMakeTool" - /> - <Tool - Name="VCBscMakeTool" - /> - <Tool - Name="VCFxCopTool" - /> - <Tool - Name="VCAppVerifierTool" - /> - <Tool - Name="VCWebDeploymentTool" - /> - <Tool - Name="VCPostBuildEventTool" - /> - </Configuration> - </Configurations> - <References> - </References> - <Files> - <Filter - Name="Header Files" - Filter="h;hpp;hxx;hm;inl;inc;xsd" - UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" - > - </Filter> - <Filter - Name="Resource Files" - Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx" - UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}" - > - <File - RelativePath=".\releases\xdelta30h.ppc-osx.bin" - > - </File> - </Filter> - <Filter - Name="Source Files" - Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" - UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" - > - <File - RelativePath=".\badcopy.c" - > - </File> - </Filter> - <File - RelativePath=".\release\BuildLog.htm" - > - </File> - <File - RelativePath=".\debug\BuildLog.htm" - > - </File> - <File - RelativePath=".\www\xdelta3-api-guide.html" - > - </File> - <File - RelativePath=".\www\xdelta3-cmdline.html" - > - </File> - <File - RelativePath=".\www\xdelta3.html" - > - </File> - </Files> - <Globals> - </Globals> -</VisualStudioProject> diff --git a/draft-korn-vcdiff.txt b/draft-korn-vcdiff.txt deleted file mode 100644 index 1487deb..0000000 --- a/draft-korn-vcdiff.txt +++ /dev/null @@ -1,1322 +0,0 @@ - David G. Korn, AT&T Labs - Joshua P. MacDonald, UC Berkeley - Jeffrey C. Mogul, Compaq WRL -Internet-Draft Kiem-Phong Vo, AT&T Labs -Expires: 09 November 2002 09 November 2001 - - - The VCDIFF Generic Differencing and Compression Data Format - - draft-korn-vcdiff-06.txt - - - -Status of this Memo - - This document is an Internet-Draft and is in full conformance - with all provisions of Section 10 of RFC2026. - - Internet-Drafts are working documents of the Internet Engineering - Task Force (IETF), its areas, and its working groups. Note that - other groups may also distribute working documents as - Internet-Drafts. - - Internet-Drafts are draft documents valid for a maximum of six - months and may be updated, replaced, or obsoleted by other - documents at any time. It is inappropriate to use Internet- - Drafts as reference material or to cite them other than as - "work in progress." - - The list of current Internet-Drafts can be accessed at - http://www.ietf.org/ietf/1id-abstracts.txt - - The list of Internet-Draft Shadow Directories can be accessed at - http://www.ietf.org/shadow.html. - - -Abstract - - This memo describes a general, efficient and portable data format - suitable for encoding compressed and/or differencing data so that - they can be easily transported among computers. - - -Table of Contents: - - 1. EXECUTIVE SUMMARY ............................................ 2 - 2. CONVENTIONS .................................................. 3 - 3. DELTA INSTRUCTIONS ........................................... 4 - 4. DELTA FILE ORGANIZATION ...................................... 5 - 5. DELTA INSTRUCTION ENCODING ................................... 9 - 6. DECODING A TARGET WINDOW ..................................... 14 - 7. APPLICATION-DEFINED CODE TABLES .............................. 16 - 8. PERFORMANCE .................................................. 16 - 9. FURTHER ISSUES ............................................... 17 - 10. SUMMARY ...................................................... 18 - 11. ACKNOWLEDGEMENTS ............................................. 18 - 12. SECURITY CONSIDERATIONS ...................................... 18 - 13. SOURCE CODE AVAILABILITY ..................................... 18 - 14. INTELLECTUAL PROPERTY RIGHTS ................................. 18 - 15. IANA CONSIDERATIONS .......................................... 19 - 16. REFERENCES ................................................... 19 - 17. AUTHOR'S ADDRESS ............................................. 20 - - -1. EXECUTIVE SUMMARY - - Compression and differencing techniques can greatly improve storage - and transmission of files and file versions. Since files are often - transported across machines with distinct architectures and performance - characteristics, such data should be encoded in a form that is portable - and can be decoded with little or no knowledge of the encoders. - This document describes Vcdiff, a compact portable encoding format - designed for these purposes. - - Data differencing is the process of computing a compact and invertible - encoding of a "target file" given a "source file". Data compression - is similar but without the use of source data. The UNIX utilities diff, - compress, and gzip are well-known examples of data differencing and - compression tools. For data differencing, the computed encoding is - called a "delta file", and, for data compression, it is called - a "compressed file". Delta and compressed files are good for storage - and transmission as they are often smaller than the originals. - - Data differencing and data compression are traditionally treated - as distinct types of data processing. However, as shown in the Vdelta - technique by Korn and Vo [1], compression can be thought of as a special - case of differencing in which the source data is empty. The basic idea - is to unify the string parsing scheme used in the Lempel-Ziv'77 style - compressors [2], and the block-move technique of Tichy [3]. Loosely - speaking, this works as follows: - - a. Concatenate source and target data. - b. Parse the data from left to right as in LZ'77 but - make sure that a parsed segment starts the target data. - c. Start to output when reaching target data. - - Parsing is based on string matching algorithms such as suffix trees [4] - or hashing with different time and space performance characteristics. - Vdelta uses a fast string matching algorithm that requires less memory - than other techniques [5,6]. However, even with this algorithm, the - memory requirement can still be prohibitive for large files. A common - way to deal with memory limitation is to partition an input file into - chunks called "windows" and process them separately. Here, except for - unpublished work by Vo, little has been done on designing effective - windowing schemes. Current techniques, including Vdelta, simply use - source and target windows with corresponding addresses across source - and target files. - - String matching and windowing algorithms have large influence on the - compression rate of delta and compressed files. However, it is desirable - to have a portable encoding format that is independent of such algorithms. - This enables construction of client-server applications in which a server - may serve clients with unknown computing characteristics. Unfortunately, - all current differencing and compressing tools, including Vdelta, fall - short in this respect. Their storage formats are closely intertwined - with the implemented string matching and/or windowing algorithms. - - The encoding format Vcdiff proposed here addresses the above issues. - Vcdiff achieves the below characteristics: - - Output compactness: - The basic encoding format compactly represents compressed or delta - files. Applications can further extend the basic encoding format - with "secondary encoders" to achieve more compression. - - Data portability: - The basic encoding format is free from machine byte order and - word size issues. This allows data to be encoded on one machine - and decoded on a different machine with different architecture. - - Algorithm genericity: - The decoding algorithm is independent from string matching and - windowing algorithms. This allows competition among implementations - of the encoder while keeping the same decoder. - - Decoding efficiency: - Except for secondary encoder issues, the decoding algorithm runs - in time proportional to the size of the target file and uses space - proportional to the maximal window size. Vcdiff differs from more - conventional compressors in that it uses only byte-aligned - data, thus avoiding bit-level operations, which improves - decoding speed at the slight cost of compression efficiency. - - The Vcdiff data format and the algorithms for decoding data shall be - described next. Since Vcdiff treats compression as a special case of - differencing, we shall use the term "delta file" to indicate the - compressed output for both cases. - - -2. CONVENTIONS - - The basic data unit is a byte. For portability, Vcdiff shall limit - a byte to its lower eight bits even on machines with larger bytes. - The bits in a byte are ordered from right to left so that the least - significant bit (LSB) has value 1, and the most significant bit (MSB), - has value 128. - - For purposes of exposition in this document, we adopt the convention - that the LSB is numbered 0, and the MSB is numbered 7. Bit numbers - never appear in the encoded format itself. - - Vcdiff encodes unsigned integer values using a portable variable-sized - format (originally introduced in the Sfio library [7]). This encoding - treats an integer as a number in base 128. Then, each digit in this - representation is encoded in the lower seven bits of a byte. Except for - the least significant byte, other bytes have their most significant bit - turned on to indicate that there are still more digits in the encoding. - The two key properties of this integer encoding that are beneficial - to a data compression format are: - - a. The encoding is portable among systems using 8-bit bytes, and - b. Small values are encoded compactly. - - For example, consider the value 123456789 which can be represented with - four 7-bit digits whose values are 58, 111, 26, 21 in order from most - to least significant. Below is the 8-bit byte encoding of these digits. - Note that the MSBs of 58, 111 and 26 are on. - - +-------------------------------------------+ - | 10111010 | 11101111 | 10011010 | 00010101 | - +-------------------------------------------+ - MSB+58 MSB+111 MSB+26 0+21 - - - Henceforth, the terms "byte" and "integer" will refer to a byte and an - unsigned integer as described. - - - From time to time, algorithms are exhibited to clarify the descriptions - of parts of the Vcdiff format. On such occasions, the C language will be - used to make precise the algorithms. The C code shown in this - document is meant for clarification only, and is not part of the - actual specification of the Vcdiff format. - - In this specification, the key words "MUST", "MUST NOT", - "SHOULD", "SHOULD NOT", and "MAY" document are to be interpreted as - described in RFC2119 [12]. - - -3. DELTA INSTRUCTIONS - - A large target file is partitioned into non-overlapping sections - called "target windows". These target windows are processed separately - and sequentially based on their order in the target file. - - A target window T of length t may be compared against some source data - segment S of length s. By construction, this source data segment S - comes either from the source file, if one is used, or from a part of - the target file earlier than T. In this way, during decoding, S is - completely known when T is being decoded. - - The choices of T, t, S and s are made by some window selection algorithm - which can greatly affect the size of the encoding. However, as seen later, - these choices are encoded so that no knowledge of the window selection - algorithm is needed during decoding. - - Assume that S[j] represents the jth byte in S, and T[k] represents - the kth byte in T. Then, for the delta instructions, we treat the data - windows S and T as substrings of a superstring U formed by concatenating - them like this: - - S[0]S[1]...S[s-1]T[0]T[1]...T[t-1] - - The "address" of a byte in S or T is referred to by its location in U. - For example, the address of T[k] is s+k. - - The instructions to encode and direct the reconstruction of a target - window are called delta instructions. There are three types: - - ADD: This instruction has two arguments, a size x and a sequence of - x bytes to be copied. - COPY: This instruction has two arguments, a size x and an address p - in the string U. The arguments specify the substring of U that - must be copied. We shall assert that such a substring must be - entirely contained in either S or T. - RUN: This instruction has two arguments, a size x and a byte b that - will be repeated x times. - - Below are example source and target windows and the delta instructions - that encode the target window in terms of the source window. - - a b c d e f g h i j k l m n o p - a b c d w x y z e f g h e f g h e f g h e f g h z z z z - - COPY 4, 0 - ADD 4, w x y z - COPY 4, 4 - COPY 12, 24 - RUN 4, z - - - Thus, the first letter 'a' in the target window is at location 16 - in the superstring. Note that the fourth instruction, "COPY 12, 24", - copies data from T itself since address 24 is position 8 in T. - This instruction also shows that it is fine to overlap the data to be - copied with the data being copied from as long as the latter starts - earlier. This enables efficient encoding of periodic sequences, - i.e., sequences with regularly repeated subsequences. The RUN instruction - is a compact way to encode a sequence repeating the same byte even though - such a sequence can be thought of as a periodic sequence with period 1. - - To reconstruct the target window, one simply processes one delta - instruction at a time and copy the data either from the source window - or the being reconstructed target window based on the type of the - instruction and the associated address, if any. - - -4. DELTA FILE ORGANIZATION - - A Vcdiff delta file starts with a Header section followed by a sequence - of Window sections. The Header section includes magic bytes to identify - the file type, and information concerning data processing beyond the - basic encoding format. The Window sections encode the target windows. - - Below is the overall organization of a delta file. The indented items - refine the ones immediately above them. An item in square brackets may - or may not be present in the file depending on the information encoded - in the Indicator byte above it. - - Header - Header1 - byte - Header2 - byte - Header3 - byte - Header4 - byte - Hdr_Indicator - byte - [Secondary compressor ID] - byte - -[@@@ Why is compressor ID not an integer? ] -[@@@ If we aren't defining any secondary compressors yet, then it seems -that defining the [Secondary compressor ID] and the corresponding -VCD_DECOMPRESS Hdr_Indicator bit in this draft has no real value. An -implementation of this specification won't be able to decode a VCDIFF -encoded with this option if it doesn't know about any secondary -compressors. It seems that you should specify the bits related to -secondary compressors once you have defined the first a secondary -compressor. I can imagine a secondary-compressor might want to supply -extra information, such as a dictionary of some kind, in which case -this speculative treatment wouldn't go far enough.] - - [Length of code table data] - integer - [Code table data] - Size of near cache - byte - Size of same cache - byte - Compressed code table data - Window1 - Win_Indicator - byte - [Source segment size] - integer - [Source segment position] - integer - The delta encoding of the target window - Length of the delta encoding - integer - The delta encoding - Size of the target window - integer - Delta_Indicator - byte - Length of data for ADDs and RUNs - integer - Length of instructions and sizes - integer - Length of addresses for COPYs - integer - Data section for ADDs and RUNs - array of bytes - Instructions and sizes section - array of bytes - Addresses section for COPYs - array of bytes - Window2 - ... - - - -4.1 The Header Section - - Each delta file starts with a header section organized as below. - Note the convention that square-brackets enclose optional items. - - Header1 - byte = 0xE6 - Header2 - byte = 0xD3 - Header3 - byte = 0xD4 - -HMMM - -0xD6 -0xC3 -0xC4 - - Header4 - byte - Hdr_Indicator - byte - [Secondary compressor ID] - byte - [Length of code table data] - integer - [Code table data] - - The first three Header bytes are the ASCII characters 'V', 'C' and 'D' - with their most significant bits turned on (in hexadecimal, the values - are 0xE6, 0xD3, and 0xD4). The fourth Header byte is currently set to - zero. In the future, it might be used to indicate the version of Vcdiff. - - The Hdr_Indicator byte shows if there are any initialization data - required to aid in the reconstruction of data in the Window sections. - This byte MAY have non-zero values for either, both, or neither of - the two bits VCD_DECOMPRESS and VCD_CODETABLE below: - - 7 6 5 4 3 2 1 0 - +-+-+-+-+-+-+-+-+ - | | | | | | | | | - +-+-+-+-+-+-+-+-+ - ^ ^ - | | - | +-- VCD_DECOMPRESS - +---- VCD_CODETABLE - - If bit 0 (VCD_DECOMPRESS) is non-zero, this indicates that a secondary - compressor may have been used to further compress certain parts of the - delta encoding data as described in Sections 4.3 and 6. In that case, - the ID of the secondary compressor is given next. If this bit is zero, - the compressor ID byte is not included. - -[@@@ If we aren't defining any secondary compressors yet, then it seems -this bit has no real value yet..] - - If bit 1 (VCD_CODETABLE) is non-zero, this indicates that an - application-defined code table is to be used for decoding the delta - instructions. This table itself is compressed. The length of the data - comprising this compressed code table and the data follow next. Section 7 - discusses application-defined code tables. If this bit is zero, the code - table data length and the code table data are not included. - - If both bits are set, then the compressor ID byte is included - before the code table data length and the code table data. - - -4.2 The Format of a Window Section - - Each Window section is organized as follows: - - Win_Indicator - byte - [Source segment length] - integer - [Source segment position] - integer - The delta encoding of the target window - - - Below are the detail of the various items: - -[@@@ Here, I want to replace the Win_Indicator with a source-count, -followed by source-count length/position pairs?] - - Win_Indicator: - This byte is a set of bits, as shown: - - 7 6 5 4 3 2 1 0 - +-+-+-+-+-+-+-+-+ - | | | | | | | | | - +-+-+-+-+-+-+-+-+ - ^ ^ - | | - | +-- VCD_SOURCE - +---- VCD_TARGET - - - If bit 0 (VCD_SOURCE) is non-zero, this indicates that a segment - of data from the "source" file was used as the corresponding - source window of data to encode the target window. The decoder - will use this same source data segment to decode the target window. - - If bit 1 (VCD_TARGET) is non-zero, this indicates that a segment - of data from the "target" file was used as the corresponding - source window of data to encode the target window. As above, this - same source data segment is used to decode the target window. - - The Win_Indicator byte MUST NOT have more than one of the bits - set (non-zero). It MAY have none of these bits set. - - If one of these bits is set, the byte is followed by two - integers to indicate respectively the length and position of - the source data segment in the relevant file. If the - indicator byte is zero, the target window was compressed - by itself without comparing against another data segment, - and these two integers are not included. - - The delta encoding of the target window: - This contains the delta encoding of the target window either - in terms of the source data segment (i.e., VCD_SOURCE - or VCD_TARGET was set) or by itself if no source window - is specified. This data format is discussed next. - - -4.3 The Delta Encoding of a Target Window - - The delta encoding of a target window is organized as follows: - - Length of the delta encoding - integer - The delta encoding - Length of the target window - integer - Delta_Indicator - byte - Length of data for ADDs and RUNs - integer - Length of instructions section - integer - Length of addresses for COPYs - integer - Data section for ADDs and RUNs - array of bytes - Instructions and sizes section - array of bytes - Addresses section for COPYs - array of bytes - - - Length of the delta encoding: - This integer gives the total number of remaining bytes that - comprise data of the delta encoding for this target window. - - The delta encoding: - This contains the data representing the delta encoding which - is described next. - - Length of the target window: - This integer indicates the actual size of the target window - after decompression. A decoder can use this value to allocate - memory to store the uncompressed data. - - Delta_Indicator: - This byte is a set of bits, as shown: - - 7 6 5 4 3 2 1 0 - +-+-+-+-+-+-+-+-+ - | | | | | | | | | - +-+-+-+-+-+-+-+-+ - ^ ^ ^ - | | | - | | +-- VCD_DATACOMP - | +---- VCD_INSTCOMP - +------ VCD_ADDRCOMP - - VCD_DATACOMP: bit value 1. - VCD_INSTCOMP: bit value 2. - VCD_ADDRCOMP: bit value 4. - - As discussed, the delta encoding consists of COPY, ADD and RUN - instructions. The ADD and RUN instructions have accompanying - unmatched data (that is, data that does not specifically match - any data in the source window or in some earlier part of the - target window) and the COPY instructions have addresses of where - the matches occur. OPTIONALLY, these types of data MAY be further - compressed using a secondary compressor. Thus, Vcdiff separates - the encoding of the delta instructions into three parts: - - a. The unmatched data in the ADD and RUN instructions, - b. The delta instructions and accompanying sizes, and - c. The addresses of the COPY instructions. - - If the bit VCD_DECOMPRESS (Section 4.1) was on, each of these - sections may have been compressed using the specified secondary - compressor. The bit positions 0 (VCD_DATACOMP), 1 (VCD_INSTCOMP), - and 2 (VCD_ADDRCOMP) respectively indicate, if non-zero, that - the corresponding parts are compressed. Then, these parts MUST - be decompressed before decoding the delta instructions. - - Length of data for ADDs and RUNs: - This is the length (in bytes) of the section of data storing - the unmatched data accompanying the ADD and RUN instructions. - - Length of instructions section: - This is the length (in bytes) of the delta instructions and - accompanying sizes. - - Length of addresses for COPYs: - This is the length (in bytes) of the section storing - the addresses of the COPY instructions. - - Data section for ADDs and RUNs: - This sequence of bytes encodes the unmatched data for the ADD - and RUN instructions. - - Instructions and sizes section: - This sequence of bytes encodes the instructions and their sizes. - - Addresses section for COPYs: - This sequence of bytes encodes the addresses of the COPY - instructions. - - -5. DELTA INSTRUCTION ENCODING - - The delta instructions described in Section 3 represent the results of - string matching. For many data differencing applications in which the - changes between source and target data are small, any straightforward - representation of these instructions would be adequate. However, for - applications including data compression, it is important to encode - these instructions well to achieve good compression rates. From our - experience, the following observations can be made: - - a. The addresses in COPY instructions are locations of matches and - often occur close by or even exactly equal to one another. This is - because data in local regions are often replicated with minor changes. - In turn, this means that coding a newly matched address against some - set of recently matched addresses can be beneficial. - - b. The matches are often short in length and separated by small amounts - of unmatched data. That is, the lengths of COPY and ADD instructions - are often small. This is particularly true of binary data such as - executable files or structured data such as HTML or XML. In such cases, - compression can be improved by combining the encoding of the sizes - and the instruction types as well as combining the encoding of adjacent - delta instructions with sufficiently small data sizes. - - The below subsections discuss how the Vcdiff data format provides - mechanisms enabling encoders to use the above observations to improve - compression rates. - - -5.1 Address Encoding Modes of COPY Instructions - - As mentioned earlier, addresses of COPY instructions often occur close - to one another or are exactly equal. To take advantage of this phenomenon - and encode addresses of COPY instructions more efficiently, the Vcdiff - data format supports the use of two different types of address caches. - Both the encoder and decoder maintain these caches, so that decoder's - caches remain synchronized with the encoder's caches. - - a. A "near" cache is an array with "s_near" slots, each containing an - address used for encoding addresses nearby to previously encoded - addresses (in the positive direction only). The near cache also - maintains a "next_slot" index to the near cache. New entries to the - near cache are always inserted in the next_slot index, which maintains - a circular buffer of the s_near most recent addresses. - - b. A "same" cache is an array with "s_same" multiple of 256 slots, each - containing an address. The same cache maintains a hash table of recent - addresses used for repeated encoding of the exact same address. - - - By default, the parameters s_near and s_same are respectively set to 4 - and 3. An encoder MAY modify these values, but then it MUST encode the - new values in the encoding itself, as discussed in Section 7, so that - the decoder can properly set up its own caches. - - At the start of processing a target window, an implementation - (encoder or decoder) initializes all of the slots in both caches - to zero. The next_slot pointer of the near cache is set - to point to slot zero. - - Each time a COPY instruction is processed by the encoder or - decoder, the implementation's caches are updated as follows, where - "addr" is the address in the COPY instruction. - - a. The slot in the near cache referenced by the next_slot - index is set to addr. The next_slot index is then incremented - modulo s_near. - - b. The slot in the same cache whose index is addr%(s_same*256) - is set to addr. [We use the C notations of % for modulo and - * for multiplication.] - - -5.2 Example code for maintaining caches - - To make clear the above description, below are example cache data - structures and algorithms to initialize and update them: - - typedef struct _cache_s - { - int* near; /* array of size s_near */ - int s_near; - int next_slot; /* the circular index for near */ - int* same; /* array of size s_same*256 */ - int s_same; - } Cache_t; - - cache_init(Cache_t* ka) - { - int i; - - ka->next_slot = 0; - for(i = 0; i < ka->s_near; ++i) - ka->near[i] = 0; - - for(i = 0; i < ka->s_same*256; ++i) - ka->same[i] = 0; - } - - cache_update(Cache_t* ka, int addr) - { - if(ka->s_near > 0) - { ka->near[ka->next_slot] = addr; - ka->next_slot = (ka->next_slot + 1) % ka->s_near; - } - - if(ka->s_same > 0) - ka->same[addr % (ka->s_same*256)] = addr; - } - - -5.3 Encoding of COPY instruction addresses - - The address of a COPY instruction is encoded using different modes - depending on the type of cached address used, if any. - - Let "addr" be the address of a COPY instruction to be decoded and "here" - be the current location in the target data (i.e., the start of the data - about to be encoded or decoded). Let near[j] be the jth element in - the near cache, and same[k] be the kth element in the same cache. - Below are the possible address modes: - - VCD_SELF: This mode has value 0. The address was encoded by itself - as an integer. - - VCD_HERE: This mode has value 1. The address was encoded as - the integer value "here - addr". - - Near modes: The "near modes" are in the range [2,s_near+1]. Let m - be the mode of the address encoding. The address was encoded - as the integer value "addr - near[m-2]". - - Same modes: The "same modes" are in the range - [s_near+2,s_near+s_same+1]. Let m be the mode of the encoding. - The address was encoded as a single byte b such that - "addr == same[(m - (s_near+2))*256 + b]". - - -5.3 Example code for encoding and decoding of COPY instruction addresses - - We show example algorithms below to demonstrate use of address modes more - clearly. The encoder has freedom to choose address modes, the sample - addr_encode() algorithm merely shows one way of picking the address - mode. The decoding algorithm addr_decode() will uniquely decode addresses - regardless of the encoder's algorithm choice. - - Note that the address caches are updated immediately after an address is - encoded or decoded. In this way, the decoder is always synchronized with - the encoder. - - int addr_encode(Cache_t* ka, int addr, int here, int* mode) - { - int i, d, bestd, bestm; - - /* Attempt to find the address mode that yields the - * smallest integer value for "d", the encoded address - * value, thereby minimizing the encoded size of the - * address. */ - - bestd = addr; bestm = VCD_SELF; /* VCD_SELF == 0 */ - - if((d = here-addr) < bestd) - { bestd = d; bestm = VCD_HERE; } /* VCD_HERE == 1 */ - - for(i = 0; i < ka->s_near; ++i) - if((d = addr - ka->near[i]) >= 0 && d < bestd) - { bestd = d; bestm = i+2; } - - if(ka->s_same > 0 && ka->same[d = addr%(ka->s_same*256)] == addr) - { bestd = d%256; bestm = ka->s_near + 2 + d/256; } - - cache_update(ka,addr); - - *mode = bestm; /* this returns the address encoding mode */ - return bestd; /* this returns the encoded address */ - } - - Note that the addr_encode() algorithm chooses the best address mode using a - local optimization, but that may not lead to the best encoding efficiency - because different modes lead to different instruction encodings, as described below. - - The functions addrint() and addrbyte() used in addr_decode() obtain from - the "Addresses section for COPYs" (Section 4.3) an integer or a byte, - respectively. These utilities will not be described here. We simply - recall that an integer is represented as a compact variable-sized string - of bytes as described in Section 2 (i.e., base 128). - - int addr_decode(Cache_t* ka, int here, int mode) - { int addr, m; - - if(mode == VCD_SELF) - addr = addrint(); - else if(mode == VCD_HERE) - addr = here - addrint(); - else if((m = mode - 2) >= 0 && m < ka->s_near) /* near cache */ - addr = ka->near[m] + addrint(); - else /* same cache */ - { m = mode - (2 + ka->s_near); - addr = ka->same[m*256 + addrbyte()]; - } - - cache_update(ka, addr); - - return addr; - } - - -5.4 Instruction Codes - - As noted, the data sizes associated with delta instructions are often - small. Thus, compression efficiency can be improved by combining the sizes - and instruction types in a single encoding, as well by combining certain - pairs of adjacent delta instructions. Effective choices of when to perform - such combinations depend on many factors including the data being processed - and the string matching algorithm in use. For example, if many COPY - instructions have the same data sizes, it may be worth to encode these - instructions more compactly than others. - - The Vcdiff data format is designed so that a decoder does not need to be - aware of the choices made in encoding algorithms. This is achieved with the - notion of an "instruction code table" containing 256 entries. Each entry - defines either a single delta instruction or a pair of instructions that - have been combined. Note that the code table itself only exists in main - memory, not in the delta file (unless using an application-defined code - table, described in Section 7). The encoded data simply includes the index - of each instruction and, since there are only 256 indices, each index - can be represented as a single byte. - - Each instruction code entry contains six fields, each of which - is a single byte with unsigned value: - - +-----------------------------------------------+ - | inst1 | size1 | mode1 | inst2 | size2 | mode2 | - +-----------------------------------------------+ - -@@@ could be more compact - - Each triple (inst,size,mode) defines a delta instruction. The meanings - of these fields are as follows: - - inst: An "inst" field can have one of the four values: NOOP (0), ADD (1), - RUN (2) or COPY (3) to indicate the instruction types. NOOP means - that no instruction is specified. In this case, both the corresponding - size and mode fields will be zero. - - size: A "size" field is zero or positive. A value zero means that the - size associated with the instruction is encoded separately as - an integer in the "Instructions and sizes section" (Section 6). - A positive value for "size" defines the actual data size. - Note that since the size is restricted to a byte, the maximum - value for any instruction with size implicitly defined in the code - table is 255. - - mode: A "mode" field is significant only when the associated delta - instruction is a COPY. It defines the mode used to encode the - associated addresses. For other instructions, this is always zero. - - -5.5 The Code Table - - Following the discussions on address modes and instruction code tables, - we define a "Code Table" to have the data below: - - s_near: the size of the near cache, - s_same: the size of the same cache, - i_code: the 256-entry instruction code table. - - Vcdiff itself defines a "default code table" in which s_near is 4 - and s_same is 3. Thus, there are 9 address modes for a COPY instruction. - The first two are VCD_SELF (0) and VCD_HERE (1). Modes 2, 3, 4 and 5 - are for addresses coded against the near cache. And, modes 6, 7 and 8 - are for addresses coded against the same cache. - - The default instruction code table is depicted below, in a compact - representation that we use only for descriptive purposes. See section 7 - for the specification of how an instruction code table is represented - in the Vcdiff encoding format. In the depiction, a zero value for - size indicates that the size is separately coded. The mode of non-COPY - instructions is represented as 0 even though they are not used. - - - TYPE SIZE MODE TYPE SIZE MODE INDEX - --------------------------------------------------------------- - 1. RUN 0 0 NOOP 0 0 0 - 2. ADD 0, [1,17] 0 NOOP 0 0 [1,18] - 3. COPY 0, [4,18] 0 NOOP 0 0 [19,34] - 4. COPY 0, [4,18] 1 NOOP 0 0 [35,50] - 5. COPY 0, [4,18] 2 NOOP 0 0 [51,66] - 6. COPY 0, [4,18] 3 NOOP 0 0 [67,82] - 7. COPY 0, [4,18] 4 NOOP 0 0 [83,98] - 8. COPY 0, [4,18] 5 NOOP 0 0 [99,114] - 9. COPY 0, [4,18] 6 NOOP 0 0 [115,130] - 10. COPY 0, [4,18] 7 NOOP 0 0 [131,146] - 11. COPY 0, [4,18] 8 NOOP 0 0 [147,162] - 12. ADD [1,4] 0 COPY [4,6] 0 [163,174] - 13. ADD [1,4] 0 COPY [4,6] 1 [175,186] - 14. ADD [1,4] 0 COPY [4,6] 2 [187,198] - 15. ADD [1,4] 0 COPY [4,6] 3 [199,210] - 16. ADD [1,4] 0 COPY [4,6] 4 [211,222] - 17. ADD [1,4] 0 COPY [4,6] 5 [223,234] - 18. ADD [1,4] 0 COPY 4 6 [235,238] - 19. ADD [1,4] 0 COPY 4 7 [239,242] - 20. ADD [1,4] 0 COPY 4 8 [243,246] - 21. COPY 4 [0,8] ADD 1 0 [247,255] - --------------------------------------------------------------- - - In the above depiction, each numbered line represents one or more - entries in the actual instruction code table (recall that an entry in - the instruction code table may represent up to two combined delta - instructions.) The last column ("INDEX") shows which index value or - range of index values of the entries covered by that line. The notation - [i,j] means values from i through j, inclusive. The first 6 columns of - a line in the depiction describe the pairs of instructions used for - the corresponding index value(s). - - If a line in the depiction includes a column entry using the [i,j] - notation, this means that the line is instantiated for each value - in the range from i to j, inclusive. The notation "0, [i,j]" means - that the line is instantiated for the value 0 and for each value - in the range from i to j, inclusive. - - If a line in the depiction includes more than one entry using the [i,j] - notation, implying a "nested loop" to convert the line to a range of - table entries, the first such [i,j] range specifies the outer loop, - and the second specifies the inner loop. - - The below examples should make clear the above description: - - Line 1 shows the single RUN instruction with index 0. As the size field - is 0, this RUN instruction always has its actual size encoded separately. - - Line 2 shows the 18 single ADD instructions. The ADD instruction with - size field 0 (i.e., the actual size is coded separately) has index 1. - ADD instructions with sizes from 1 to 17 use code indices 2 to 18 and - their sizes are as given (so they will not be separately encoded.) - - Following the single ADD instructions are the single COPY instructions - ordered by their address encoding modes. For example, line 11 shows the - COPY instructions with mode 8, i.e., the last of the same cache. - In this case, the COPY instruction with size field 0 has index 147. - Again, the actual size of this instruction will be coded separately. - - Lines 12 to 21 show the pairs of instructions that are combined together. - For example, line 12 depicts the 12 entries in which an ADD instruction - is combined with an immediately following COPY instruction. The entries - with indices 163, 164, 165 represent the pairs in which the ADD - instructions all have size 1 while the COPY instructions has mode - 0 (VCD_SELF) and sizes 4, 5 and 6 respectively. - - The last line, line 21, shows the eight instruction pairs where the first - instruction is a COPY and the second is an ADD. In this case, all COPY - instructions have size 4 with mode ranging from 0 to 8 and all the ADD - instructions have size 1. Thus, the entry with largest index 255 - combines a COPY instruction of size 4 and mode 8 with an ADD instruction - of size 1. - - The choice of the minimum size 4 for COPY instructions in the default code - table was made from experiments that showed that excluding small matches - (less then 4 bytes long) improved the compression rates. - - -6. DECODING A TARGET WINDOW - - Section 4.3 discusses that the delta instructions and associated data - are encoded in three arrays of bytes: - - Data section for ADDs and RUNs, - Instructions and sizes section, and - Addresses section for COPYs. - - - Further, these data sections may have been further compressed by some - secondary compressor. Assuming that any such compressed data has been - decompressed so that we now have three arrays: - - inst: bytes coding the instructions and sizes. - data: unmatched data associated with ADDs and RUNs. - addr: bytes coding the addresses of COPYs. - - These arrays are organized as follows: - - inst: - a sequence of (index, [size1], [size2]) tuples, where "index" - is an index into the instruction code table, and size1 and size2 - are integers that MAY or MAY NOT be included in the tuple as - follows. The entry with the given "index" in the instruction - code table potentially defines two delta instructions. If the - first delta instruction is not a VCD_NOOP and its size is zero, - then size1 MUST be present. Otherwise, size1 MUST be omitted and - the size of the instruction (if it is not VCD_NOOP) is as defined - in the table. The presence or absence of size2 is defined - similarly with respect to the second delta instruction. - - data: - a sequence of data values, encoded as bytes. - - addr: - a sequence of address values. Addresses are normally encoded as - integers as described in Section 2 (i.e., base 128). - Since the same cache emits addresses in the range [0,255], - however, same cache addresses are always encoded as a - single byte. - - To summarize, each tuple in the "inst" array includes an index to some - entry in the instruction code table that determines: - - a. Whether one or two instructions were encoded and their types. - - b. If the instructions have their sizes encoded separately, these - sizes will follow, in order, in the tuple. - - c. If the instructions have accompanying data, i.e., ADDs or RUNs, - their data will be in the array "data". - - d. Similarly, if the instructions are COPYs, the coded addresses are - found in the array "addr". - - The decoding procedure simply processes the arrays by reading one code - index at a time, looking up the corresponding instruction code entry, - then consuming the respective sizes, data and addresses following the - directions in this entry. In other words, the decoder maintains an implicit - next-element pointer for each array; "consuming" an instruction tuple, - data, or address value implies incrementing the associated pointer. - - For example, if during the processing of the target window, the next - unconsumed tuple in the inst array has index value 19, then the first - instruction is a COPY, whose size is found as the immediately following - integer in the inst array. Since the mode of this COPY instruction is - VCD_SELF, the corresponding address is found by consuming the next - integer in the addr array. The data array is left intact. As the second - instruction for code index 19 is a NOOP, this tuple is finished. - - -7. APPLICATION-DEFINED CODE TABLES - - Although the default code table used in Vcdiff is good for general - purpose encoders, there are times when other code tables may perform - better. For example, to code a file with many identical segments of data, - it may be advantageous to have a COPY instruction with the specific size - of these data segments so that the instruction can be encoded in a single - byte. Such a special code table MUST then be encoded in the delta file - so that the decoder can reconstruct it before decoding the data. - - Vcdiff allows an application-defined code table to be specified - in a delta file with the following data: - - Size of near cache - byte - Size of same cache - byte - Compressed code table data - - The "compressed code table data" encodes the delta between the default - code table (source) and the new code table (target) in the same manner as - described in Section 4.3 for encoding a target window in terms of a - source window. This delta is computed using the following steps: - - a. Convert the new instruction code table into a string, "code", of - 1536 bytes using the below steps in order: - - i. Add in order the 256 bytes representing the types of the first - instructions in the instruction pairs. - ii. Add in order the 256 bytes representing the types of the second - instructions in the instruction pairs. - iii. Add in order the 256 bytes representing the sizes of the first - instructions in the instruction pairs. - iv. Add in order the 256 bytes representing the sizes of the second - instructions in the instruction pairs. - v. Add in order the 256 bytes representing the modes of the first - instructions in the instruction pairs. - vi. Add in order the 256 bytes representing the modes of the second - instructions in the instruction pairs. - - b. Similarly, convert the default instruction code table into - a string "dflt". - - c. Treat the string "code" as a target window and "dflt" as the - corresponding source data and apply an encoding algorithm to - compute the delta encoding of "code" in terms of "dflt". - This computation MUST use the default code table for encoding - the delta instructions. - - The decoder can then reverse the above steps to decode the compressed - table data using the method of Section 6, employing the default code - table, to generate the new code table. Note that the decoder does not - need to know anything about the details of the encoding algorithm used - in step (c). The decoder is still able to decode the new code table - because the Vcdiff format is independent from the choice of encoding - algorithm, and because the encoder in step (c) uses the known, default - code table. - - -8. PERFORMANCE - - The encoding format is compact. For compression only, using the LZ-77 - string parsing strategy and without any secondary compressors, the typical - compression rate is better than Unix compress and close to gzip. For - differencing, the data format is better than all known methods in - terms of its stated goal, which is primarily decoding speed and - encoding efficiency. - - We compare the performance of compress, gzip and Vcdiff using the - archives of three versions of the Gnu C compiler, gcc-2.95.1.tar, - gcc-2.95.2.tar and gcc-2.95.3.tar. The experiments were done on an - SGI-MIPS3, 400MHZ. Gzip was used at its default compression level. - Vcdiff timings were done using the Vcodex/Vcdiff software (Section 13). - As string and window matching typically dominates the computation during - compression, the Vcdiff compression times were directly due to the - algorithms used in the Vcodex/Vcdiff software. However, the decompression - times should be generic and representative of any good implementation - of the Vcdiff data format. Timing was done by running each program - three times and taking the average of the total cpu+system times. - - Below are the different Vcdiff runs: - - Vcdiff: vcdiff is used as compressor only. - - Vcdiff-d: vcdiff is used as a differencer only. That is, it only - compares target data against source data. Since the files - involved are large, they are broken into windows. In this - case, each target window starting at some file offset in - the target file is compared against a source window with - the same file offset (in the source file). The source - window is also slightly larger than the target window - to increase matching opportunities. The -d option also gives - a hint to the string matching algorithm of Vcdiff that - the two files are very similar with long stretches of matches. - The algorithm takes advantage of this to minimize its - processing of source data and save time. - - Vcdiff-dc: This is similar to Vcdiff-d but vcdiff can also compare - target data against target data as applicable. Thus, vcdiff - both computes differences and compresses data. The windowing - algorithm is the same as above. However, the above hint is - recinded in this case. - - Vcdiff-dcs: This is similar to Vcdiff-dc but the windowing algorithm - uses a content-based heuristic to select source data segments - that are more likely to match with a given target window. - Thus, the source data segment selected for a target window - often will not be aligned with the file offsets of this - target window. - - - gcc-2.95.1 gcc-2.95.2 compression decompression - raw size 55746560 55797760 - compress - 19939390 13.85s 7.09s - gzip - 12973443 42.99s 5.35s - Vcdiff - 15358786 20.04s 4.65s - Vcdiff-d - 100971 10.93s 1.92s - Vcdiff-dc - 97246 20.03s 1.84s - Vcdiff-dcs - 256445 44.81s 1.84s - - TABLE 1. Compressing gcc-2.95.2.tar given gcc-2.95.1 - - - TABLE 1 shows the raw sizes of gcc-2.95.1.tar and gcc-2.95.2.tar and the - sizes of the compressed results. As a pure compressor, the compression - rate for Vcdiff is worse than gzip and better than compress. The last - three rows shows that when two file versions are very similar, differencing - can have dramatically good compression rates. Vcdiff-d and Vcdiff-dc use - the same simple window selection method but Vcdiff-dc also does compression - so its output is slightly smaller. Vcdiff-dcs uses a heuristic based on - data content to search for source data that likely will match a given target - window. Although it does a good job, the heuristic did not always find the - best matches which are given by the simple algorithm of Vcdiff-d. As a - result, the output size is slightly larger. Note also that there is a large - cost in computing matching windows this way. Finally, the compression times - of Vcdiff-d is nearly half of that of Vcdiff-dc. It is tempting to conclude - that the compression feature causes the additional time in Vcdiff-dc - relative to Vcdiff-d. However, this is not the case. The hint given to - the Vcdiff string matching algorithm that the two files are likely to - have very long stretches of matches helps the algorithm to minimize - processing of the "source data", thus saving half the time. However, as we - shall see below when this hint is wrong, the result is even longer time. - - - gcc-2.95.2 gcc-2.95.3 compression decompression - raw size 55797760 55787520 - compress - 19939453 13.54s 7.00s - gzip - 12998097 42.63s 5.62s - Vcdiff - 15371737 20.09s 4.74s - Vcdiff-d - 26383849 71.41s 6.41s - Vcdiff-dc - 14461203 42.48s 4.82s - Vcdiff-dcs - 1248543 61.18s 1.99s - - TABLE 2. Compressing gcc-2.95.3.tar given gcc-2.95.2 - - - TABLE 2 shows the raw sizes of gcc-2.95.2.tar and gcc-2.95.3.tar and - the sizes of the compressed results. In this case, the tar file of - gcc-2.95.3 is rearranged in a way that makes the straightforward method - of matching file offsets for source and target windows fail. As a - result, Vcdiff-d performs rather dismally both in time and output size. - The large time for Vcdiff-d is directly due to fact that the string - matching algorithm has to work much harder to find matches when the hint - that two files have long matching stretches fails to hold. On the other - hand, Vcdiff-dc does both differencing and compression resulting in good - output size. Finally, the window searching heuristic used in Vcdiff-dcs is - effective in finding the right matching source windows for target windows - resulting a small output size. This shows why the data format needs to - have a way to specify matching windows to gain performance. Finally, - we note that the decoding times are always good regardless of how - the string matching or window searching algorithms perform. - - -9. FURTHER ISSUES - - This document does not address a few issues: - - Secondary compressors: - As discussed in Section 4.3, certain sections in the delta encoding - of a window may be further compressed by a secondary compressor. - In our experience, the basic Vcdiff format is adequate for most - purposes so that secondary compressors are seldom needed. In - particular, for normal use of data differencing where the files to - be compared have long stretches of matches, much of the gain in - compression rate is already achieved by normal string matching. - Thus, the use of secondary compressors is seldom needed in this case. - However, for applications beyond differencing of such nearly identical - files, secondary compressors may be needed to achieve maximal - compressed results. - - Therefore, we recommend to leave the Vcdiff data format defined - as in this document so that the use of secondary compressors - can be implemented when they become needed in the future. - The formats of the compressed data via such compressors or any - compressors that may be defined in the future are left open to - their implementations. These could include Huffman encoding, - arithmetic encoding, and splay tree encoding [8,9]. - - Large file system vs. small file system: - As discussed in Section 4, a target window in a large file may be - compared against some source window in another file or in the same - file (from some earlier part). In that case, the file offset of the - source window is specified as a variable-sized integer in the delta - encoding. There is a possibility that the encoding was computed on - a system supporting much larger files than in a system where - the data may be decoded (e.g., 64-bit file systems vs. 32-bit file - systems). In that case, some target data may not be recoverable. - This problem could afflict any compression format, and ought - to be resolved with a generic negotiation mechanism in the - appropriate protocol(s). - - -10. SUMMARY - - We have described Vcdiff, a general and portable encoding format for - compression and differencing. The format is good in that it allows - implementing a decoder without knowledge of the encoders. Further, - ignoring the use of secondary compressors not defined within the format, - the decoding algorithms runs in linear time and requires working space - proportional to window sizes. - - - -11. ACKNOWLEDGEMENTS - - Thanks are due to Balachander Krishnamurthy, Jeff Mogul and Arthur Van Hoff - who provided much encouragement to publicize Vcdiff. In particular, Jeff - helped clarifying the description of the data format presented here. - - - -12. SECURITY CONSIDERATIONS - - Vcdiff only provides a format to encode compressed and differenced data. - It does not address any issues concerning how such data are, in fact, - stored in a given file system or the run-time memory of a computer system. - Therefore, we do not anticipate any security issues with respect to Vcdiff. - - - -13. SOURCE CODE AVAILABILITY - - Vcdiff is implemented as a data transforming method in Phong Vo's - Vcodex library. AT&T Corp. has made the source code for Vcodex available - for anyone to use to transmit data via HTTP/1.1 Delta Encoding [10,11]. - The source code and according license is accessible at the below URL: - - http://www.research.att.com/sw/tools - - -14. INTELLECTUAL PROPERTY RIGHTS - - The IETF has been notified of intellectual property rights claimed in - regard to some or all of the specification contained in this - document. For more information consult the online list of claimed - rights, at <http://www.ietf.org/ipr.html>. - - The IETF takes no position regarding the validity or scope of any - intellectual property or other rights that might be claimed to - pertain to the implementation or use of the technology described in - this document or the extent to which any license under such rights - might or might not be available; neither does it represent that it - has made any effort to identify any such rights. Information on the - IETF's procedures with respect to rights in standards-track and - standards-related documentation can be found in BCP-11. Copies of - claims of rights made available for publication and any assurances of - licenses to be made available, or the result of an attempt made to - obtain a general license or permission for the use of such - proprietary rights by implementors or users of this specification can - be obtained from the IETF Secretariat. - - - -15. IANA CONSIDERATIONS - - The Internet Assigned Numbers Authority (IANA) administers the number - space for Secondary Compressor ID values. Values and their meaning - must be documented in an RFC or other peer-reviewed, permanent, and - readily available reference, in sufficient detail so that - interoperability between independent implementations is possible. - Subject to these constraints, name assignments are First Come, First - Served - see RFC2434 [13]. Legal ID values are in the range 1..255. - - This document does not define any values in this number space. - - -16. REFERENCES - - [1] D.G. Korn and K.P. Vo, Vdelta: Differencing and Compression, - Practical Reusable Unix Software, Editor B. Krishnamurthy, - John Wiley & Sons, Inc., 1995. - - [2] J. Ziv and A. Lempel, A Universal Algorithm for Sequential Data - Compression, IEEE Trans. on Information Theory, 23(3):337-343, 1977. - - [3] W. Tichy, The String-to-String Correction Problem with Block Moves, - ACM Transactions on Computer Systems, 2(4):309-321, November 1984. - - [4] E.M. McCreight, A Space-Economical Suffix Tree Construction - Algorithm, Journal of the ACM, 23:262-272, 1976. - - [5] J.J. Hunt, K.P. Vo, W. Tichy, An Empirical Study of Delta Algorithms, - IEEE Software Configuration and Maintenance Workshop, 1996. - - [6] J.J. Hunt, K.P. Vo, W. Tichy, Delta Algorithms: An Empirical Analysis, - ACM Trans. on Software Engineering and Methodology, 7:192-214, 1998. - - [7] D.G. Korn, K.P. Vo, Sfio: A buffered I/O Library, - Proc. of the Summer '91 Usenix Conference, 1991. - - [8] D. W. Jones, Application of Splay Trees to Data Compression, - CACM, 31(8):996:1007. - - [9] M. Nelson, J. Gailly, The Data Compression Book, ISBN 1-55851-434-1, - M&T Books, New York, NY, 1995. - - [10] J.C. Mogul, F. Douglis, A. Feldmann, and B. Krishnamurthy, - Potential benefits of delta encoding and data compression for HTTP, - SIGCOMM '97, Cannes, France, 1997. - - [11] J.C. Mogul, B. Krishnamurthy, F. Douglis, A. Feldmann, - Y. Goland, and A. Van Hoff, Delta Encoding in HTTP, - IETF, draft-mogul-http-delta-10, 2001. - - [12] S. Bradner, Key words for use in RFCs to Indicate Requirement Levels, - RFC 2119, March 1997. - - [13] T. Narten, H. Alvestrand, Guidelines for Writing an IANA - Considerations Section in RFCs, RFC2434, October 1998. - - - -17. AUTHOR'S ADDRESS - - Kiem-Phong Vo (main contact) - AT&T Labs, Room D223 - 180 Park Avenue - Florham Park, NJ 07932 - Email: kpv@research.att.com - Phone: 1 973 360 8630 - - David G. Korn - AT&T Labs, Room D237 - 180 Park Avenue - Florham Park, NJ 07932 - Email: dgk@research.att.com - Phone: 1 973 360 8602 - - Jeffrey C. Mogul - Western Research Laboratory - Compaq Computer Corporation - 250 University Avenue - Palo Alto, California, 94305, U.S.A. - Email: JeffMogul@acm.org - Phone: 1 650 617 3304 (email preferred) - - Joshua P. MacDonald - Computer Science Division - University of California, Berkeley - 345 Soda Hall - Berkeley, CA 94720 - Email: jmacd@cs.berkeley.edu diff --git a/examples/Makefile b/examples/Makefile deleted file mode 100755 index b21ebda..0000000 --- a/examples/Makefile +++ /dev/null @@ -1,32 +0,0 @@ -CFLAGS = -g -Wall -I.. -DXD3_DEBUG=1 -DNDEBUG=0 -#CFLAGS = -O3 -Wall -I.. -DXD3_DEBUG=0 -fno-builtin -DNDEBUG=1 -# -pg - -SOURCES = small_page_test.c encode_decode_test.c speed_test.c - -DEPS = ../*.h ../*.c *.h - -TARGETS = small_page_test encode_decode_test speed_test32 speed_test64 compare_test checksum_test - -all: $(TARGETS) - -small_page_test: small_page_test.c $(DEPS) - $(CC) $(CFLAGS) small_page_test.c -o small_page_test -DXD3_USE_LARGEFILE64=0 -DSECONDARY_DJW=1 - -encode_decode_test: encode_decode_test.c $(DEPS) - $(CC) $(CFLAGS) encode_decode_test.c -o encode_decode_test - -speed_test32: speed_test.c $(DEPS) - $(CC) $(CFLAGS) -DXD3_USE_LARGEFILE64=0 speed_test.c -o speed_test32 - -speed_test64: speed_test.c $(DEPS) - $(CC) $(CFLAGS) -DXD3_USE_LARGEFILE64=1 speed_test.c -o speed_test64 - -compare_test: compare_test.c - $(CC) $(CFLAGS) compare_test.c -o compare_test - -checksum_test: checksum_test.cc - $(CXX) $(CFLAGS) checksum_test.cc -o checksum_test - -clean: - rm -f *.exe *.stackdump $(TARGETS) diff --git a/examples/README b/examples/README deleted file mode 100644 index 60840bf..0000000 --- a/examples/README +++ /dev/null @@ -1,8 +0,0 @@ -Files in this directory demonstrate how to use the Xdelta3 API. Copyrights -are held by the respective authors and these files are not covered by the GPL. - -small_page_test.c -- how to use xdelta3 in an environment such as the kernel -for small pages with little memory - -encode_decode_test.c -- how to use xdelta3 to process (encode/decode) data in -multiple windows with the non-blocking API diff --git a/examples/checksum_test.cc b/examples/checksum_test.cc deleted file mode 100644 index ab3ef6c..0000000 --- a/examples/checksum_test.cc +++ /dev/null @@ -1,731 +0,0 @@ -/* Copyright (C) 2007 Josh MacDonald */ - -extern "C" { -#include "test.h" -} - -#include <list> -#include <vector> -#include <map> -#include <algorithm> - -using std::list; -using std::map; -using std::vector; - -// MLCG parameters -// a, a* -uint32_t good_32bit_values[] = { - 1597334677U, // ... - 741103597U, 887987685U, -}; - -// a, a* -uint64_t good_64bit_values[] = { - 1181783497276652981ULL, 4292484099903637661ULL, - 7664345821815920749ULL, // ... -}; - -struct true_type { }; -struct false_type { }; - -template <typename Word> -int bitsof(); - -template<> -int bitsof<uint32_t>() { - return 32; -} - -template<> -int bitsof<uint64_t>() { - return 64; -} - -struct plain { - int operator()(const uint8_t &c) { - return c; - } -}; - -template <typename Word> -struct hhash { // take "h" of the high-bits as a hash value for this - // checksum, which are the most "distant" in terms of the - // spectral test for the rabin_karp MLCG. For short windows, - // the high bits aren't enough, XOR "mask" worth of these in. - Word operator()(const Word& t, const int &h, const int &mask) { - return (t >> h) ^ (t & mask); - } -}; - -template <typename Word> -Word good_word(); - -template<> -uint32_t good_word<uint32_t>() { - return good_32bit_values[0]; -} - -template<> -uint64_t good_word<uint64_t>() { - return good_64bit_values[0]; -} - -// CLASSES - -#define SELF Word, CksumSize, CksumSkip, Permute, Hash, Compaction -#define MEMBER template <typename Word, \ - int CksumSize, \ - int CksumSkip, \ - typename Permute, \ - typename Hash, \ - int Compaction> - -MEMBER -struct cksum_params { - typedef Word word_type; - typedef Permute permute_type; - typedef Hash hash_type; - - enum { cksum_size = CksumSize, - cksum_skip = CksumSkip, - compaction = Compaction, - }; -}; - - -MEMBER -struct rabin_karp { - typedef Word word_type; - typedef Permute permute_type; - typedef Hash hash_type; - - enum { cksum_size = CksumSize, - cksum_skip = CksumSkip, - compaction = Compaction, - }; - - // (a^cksum_size-1 c_0) + (a^cksum_size-2 c_1) ... - rabin_karp() { - multiplier = good_word<Word>(); - powers = new Word[cksum_size]; - powers[cksum_size - 1] = 1; - for (int i = cksum_size - 2; i >= 0; i--) { - powers[i] = powers[i + 1] * multiplier; - } - product = powers[0] * multiplier; - } - - ~rabin_karp() { - delete [] powers; - } - - Word step(const uint8_t *ptr) { - Word h = 0; - for (int i = 0; i < cksum_size; i++) { - h += permute_type()(ptr[i]) * powers[i]; - } - return h; - } - - Word state0(const uint8_t *ptr) { - incr_state = step(ptr); - return incr_state; - } - - Word incr(const uint8_t *ptr) { - incr_state = multiplier * incr_state - - product * permute_type()(ptr[-1]) + - permute_type()(ptr[cksum_size - 1]); - return incr_state; - } - - Word *powers; - Word product; - Word multiplier; - Word incr_state; -}; - -MEMBER -struct adler32_cksum { - typedef Word word_type; - typedef Permute permute_type; - typedef Hash hash_type; - - enum { cksum_size = CksumSize, - cksum_skip = CksumSkip, - compaction = Compaction, - }; - - Word step(const uint8_t *ptr) { - return xd3_lcksum (ptr, cksum_size); - } - - Word state0(const uint8_t *ptr) { - incr_state = step(ptr); - return incr_state; - } - - Word incr(const uint8_t *ptr) { - incr_state = xd3_large_cksum_update (incr_state, ptr - 1, cksum_size); - return incr_state; - } - - Word incr_state; -}; - -// TESTS - -template <typename Word> -struct file_stats { - typedef list<const uint8_t*> ptr_list; - typedef Word word_type; - typedef map<word_type, ptr_list> table_type; - typedef typename table_type::iterator table_iterator; - typedef typename ptr_list::iterator ptr_iterator; - - int cksum_size; - int cksum_skip; - int unique; - int unique_values; - int count; - table_type table; - - file_stats(int size, int skip) - : cksum_size(size), - cksum_skip(skip), - unique(0), - unique_values(0), - count(0) { - } - - void reset() { - unique = 0; - unique_values = 0; - count = 0; - table.clear(); - } - - void update(const word_type &word, const uint8_t *ptr) { - table_iterator t_i = table.find(word); - - count++; - - if (t_i == table.end()) { - table.insert(make_pair(word, ptr_list())); - } - - ptr_list &pl = table[word]; - - for (ptr_iterator p_i = pl.begin(); - p_i != pl.end(); - ++p_i) { - if (memcmp(*p_i, ptr, cksum_size) == 0) { - return; - } - } - - unique++; - pl.push_back(ptr); - } - - void freeze() { - unique_values = table.size(); - table.clear(); - } -}; - -struct test_result_base; - -static vector<test_result_base*> all_tests; - -struct test_result_base { - virtual ~test_result_base() { - } - virtual void reset() = 0; - virtual void print() = 0; - virtual void get(const uint8_t* buf, const int buf_size, int iters) = 0; - virtual void stat() = 0; - virtual int count() = 0; - virtual int dups() = 0; - virtual double uniqueness() = 0; - virtual double fullness() = 0; - virtual double collisions() = 0; - virtual double coverage() = 0; - virtual double compression() = 0; - virtual double time() = 0; - virtual double score() = 0; - virtual void set_score(double min_dups_frac, double min_time) = 0; - virtual double total_time() = 0; - virtual int total_count() = 0; - virtual int total_dups() = 0; -}; - -struct compare_h { - bool operator()(test_result_base *a, - test_result_base *b) { - return a->score() < b->score(); - } -}; - -MEMBER -struct test_result : public test_result_base { - typedef Word word_type; - typedef Permute permute_type; - typedef Hash hash_type; - - enum { cksum_size = CksumSize, - cksum_skip = CksumSkip, - compaction = Compaction, - }; - - const char *test_name; - file_stats<Word> fstats; - int test_size; - int n_steps; - int n_incrs; - int s_bits; - int s_mask; - int t_entries; - int h_bits; - int h_buckets_full; - double h_score; - char *hash_table; - long accum_millis; - int accum_iters; - - // These are not reset - double accum_time; - int accum_count; - int accum_dups; - int accum_colls; - int accum_size; - - test_result(const char *name) - : test_name(name), - fstats(cksum_size, cksum_skip), - hash_table(NULL), - accum_millis(0), - accum_iters(0), - accum_time(0.0), - accum_count(0), - accum_dups(0), - accum_colls(0), - accum_size(0) { - all_tests.push_back(this); - } - - ~test_result() { - reset(); - } - - void reset() { - // size of file - test_size = -1; - - // count - n_steps = -1; - n_incrs = -1; - - // four values used by new_table()/summarize_table() - s_bits = -1; - s_mask = -1; - t_entries = -1; - h_bits = -1; - h_buckets_full = -1; - - accum_millis = 0; - accum_iters = 0; - - fstats.reset(); - - // temporary - if (hash_table) { - delete(hash_table); - hash_table = NULL; - } - } - - int count() { - if (cksum_skip == 1) { - return n_incrs; - } else { - return n_steps; - } - } - - int dups() { - return fstats.count - fstats.unique; - } - - int colls() { - return fstats.unique - fstats.unique_values; - } - - double uniqueness() { - return 1.0 - (double) dups() / count(); - } - - double fullness() { - return (double) h_buckets_full / (1 << h_bits); - } - - double collisions() { - return (double) colls() / fstats.unique; - } - - double coverage() { - return (double) h_buckets_full / uniqueness() / count(); - } - - double compression() { - return 1.0 - coverage(); - } - - double time() { - return (double) accum_millis / accum_iters; - } - - double score() { - return h_score; - } - - void set_score(double min_compression, double min_time) { - h_score = (compression() - 0.99 * min_compression) - * (time() - 0.99 * min_time); - } - - double total_time() { - return accum_time; - } - - int total_count() { - return accum_count; - } - - int total_dups() { - return accum_dups; - } - - int total_colls() { - return accum_dups; - } - - void stat() { - accum_time += time(); - accum_count += count(); - accum_dups += dups(); - accum_colls += colls(); - accum_size += test_size; - } - - void print() { - if (fstats.count != count()) { - fprintf(stderr, "internal error: %d != %d\n", fstats.count, count()); - abort(); - } - printf("%s: (%u#%u) count %u uniq %0.2f%% full %u (%0.4f%% coll %0.4f%%) covers %0.2f%% w/ 2^%d @ %.4f MB/s %u iters\n", - test_name, - cksum_size, - cksum_skip, - count(), - 100.0 * uniqueness(), - h_buckets_full, - 100.0 * fullness(), - 100.0 * collisions(), - 100.0 * coverage(), - h_bits, - 0.001 * accum_iters * test_size / accum_millis, - accum_iters); - } - - int size_log2 (int slots) - { - int bits = bitsof<word_type>() - 1; - int i; - - for (i = 3; i <= bits; i += 1) { - if (slots <= (1 << i)) { - return i - compaction; - } - } - - return bits; - } - - void new_table(int entries) { - t_entries = entries; - h_bits = size_log2(entries); - - int n = 1 << h_bits; - - s_bits = bitsof<word_type>() - h_bits; - s_mask = n - 1; - - hash_table = new char[n / 8]; - memset(hash_table, 0, n / 8); - } - - int get_table_bit(int i) { - return hash_table[i/8] & (1 << i%8); - } - - int set_table_bit(int i) { - return hash_table[i/8] |= (1 << i%8); - } - - void summarize_table() { - int n = 1 << h_bits; - int f = 0; - for (int i = 0; i < n; i++) { - if (get_table_bit(i)) { - f++; - } - } - h_buckets_full = f; - } - - void get(const uint8_t* buf, const int buf_size, int test_iters) { - rabin_karp<SELF> test; - //adler32_cksum<SELF> test; - hash_type hash; - const uint8_t *ptr; - const uint8_t *end; - int last_offset; - int periods; - int stop; - - test_size = buf_size; - last_offset = buf_size - cksum_size; - - if (last_offset < 0) { - periods = 0; - n_steps = 0; - n_incrs = 0; - stop = -cksum_size; - } else { - periods = last_offset / cksum_skip; - n_steps = periods + 1; - n_incrs = last_offset + 1; - stop = last_offset - (periods + 1) * cksum_skip; - } - - // Compute file stats once. - if (fstats.unique_values == 0) { - if (cksum_skip == 1) { - for (int i = 0; i <= buf_size - cksum_size; i++) { - fstats.update(hash(test.step(buf + i), s_bits, s_mask), buf + i); - } - } else { - ptr = buf + last_offset; - end = buf + stop; - - for (; ptr != end; ptr -= cksum_skip) { - fstats.update(hash(test.step(ptr), s_bits, s_mask), ptr); - } - } - fstats.freeze(); - } - - long start_test = get_millisecs_now(); - - if (cksum_skip != 1) { - new_table(n_steps); - - for (int i = 0; i < test_iters; i++) { - ptr = buf + last_offset; - end = buf + stop; - - for (; ptr != end; ptr -= cksum_skip) { - set_table_bit(hash(test.step(ptr), s_bits, s_mask)); - } - } - - summarize_table(); - } - - stop = buf_size - cksum_size + 1; - if (stop < 0) { - stop = 0; - } - - if (cksum_skip == 1) { - - new_table(n_incrs); - - for (int i = 0; i < test_iters; i++) { - ptr = buf; - end = buf + stop; - - if (ptr != end) { - set_table_bit(hash(test.state0(ptr++), s_bits, s_mask)); - } - - for (; ptr != end; ptr++) { - Word w = test.incr(ptr); - assert(w == test.step(ptr)); - set_table_bit(hash(w, s_bits, s_mask)); - } - } - - summarize_table(); - } - - accum_iters += test_iters; - accum_millis += get_millisecs_now() - start_test; - } -}; - -template <typename Word> -void print_array(const char *tname) { - printf("static const %s hash_multiplier[64] = {\n", tname); - Word p = 1; - for (int i = 0; i < 64; i++) { - printf(" %uU,\n", p); - p *= good_word<Word>(); - } - printf("};\n", tname); -} - -int main(int argc, char** argv) { - int i; - uint8_t *buf = NULL; - size_t buf_len = 0; - int ret; - - if (argc <= 1) { - fprintf(stderr, "usage: %s file ...\n", argv[0]); - return 1; - } - - //print_array<uint32_t>("uint32_t"); - -#define TEST(T,Z,S,P,H,C) test_result<T,Z,S,P,H<T>,C> \ - _ ## T ## _ ## Z ## _ ## S ## _ ## P ## _ ## H ## _ ## C \ - (#T "_" #Z "_" #S "_" #P "_" #H "_" #C) - -#if 0 - - TEST(uint32_t, 4, SKIP, plain, hhash, 0); /* x */ \ - TEST(uint32_t, 4, SKIP, plain, hhash, 1); /* x */ \ - TEST(uint32_t, 4, SKIP, plain, hhash, 2); /* x */ \ - TEST(uint32_t, 4, SKIP, plain, hhash, 3); /* x */ \ - -#endif - -#define TESTS(SKIP) \ - TEST(uint32_t, 9, SKIP, plain, hhash, 0); /* x */ \ - TEST(uint32_t, 9, SKIP, plain, hhash, 1); /* x */ \ - TEST(uint32_t, 9, SKIP, plain, hhash, 2); /* x */ \ - TEST(uint32_t, 9, SKIP, plain, hhash, 3) - -#define TESTS_ALL(SKIP) \ - TEST(uint32_t, 3, SKIP, plain, hhash, 0); \ - TEST(uint32_t, 3, SKIP, plain, hhash, 1); \ - TEST(uint32_t, 4, SKIP, plain, hhash, 0); /* x */ \ - TEST(uint32_t, 4, SKIP, plain, hhash, 1); /* x */ \ - TEST(uint32_t, 4, SKIP, plain, hhash, 2); /* x */ \ - TEST(uint32_t, 4, SKIP, plain, hhash, 3); /* x */ \ - TEST(uint32_t, 5, SKIP, plain, hhash, 0); \ - TEST(uint32_t, 5, SKIP, plain, hhash, 1); \ - TEST(uint32_t, 8, SKIP, plain, hhash, 0); \ - TEST(uint32_t, 8, SKIP, plain, hhash, 1); \ - TEST(uint32_t, 9, SKIP, plain, hhash, 0); /* x */ \ - TEST(uint32_t, 9, SKIP, plain, hhash, 1); /* x */ \ - TEST(uint32_t, 9, SKIP, plain, hhash, 2); /* x */ \ - TEST(uint32_t, 9, SKIP, plain, hhash, 3); /* x */ \ - TEST(uint32_t, 11, SKIP, plain, hhash, 0); /* x */ \ - TEST(uint32_t, 11, SKIP, plain, hhash, 1); /* x */ \ - TEST(uint32_t, 13, SKIP, plain, hhash, 0); \ - TEST(uint32_t, 13, SKIP, plain, hhash, 1); \ - TEST(uint32_t, 15, SKIP, plain, hhash, 0); /* x */ \ - TEST(uint32_t, 15, SKIP, plain, hhash, 1); /* x */ \ - TEST(uint32_t, 16, SKIP, plain, hhash, 0); /* x */ \ - TEST(uint32_t, 16, SKIP, plain, hhash, 1); /* x */ \ - TEST(uint32_t, 21, SKIP, plain, hhash, 0); \ - TEST(uint32_t, 21, SKIP, plain, hhash, 1); \ - TEST(uint32_t, 34, SKIP, plain, hhash, 0); \ - TEST(uint32_t, 34, SKIP, plain, hhash, 1); \ - TEST(uint32_t, 55, SKIP, plain, hhash, 0); \ - TEST(uint32_t, 55, SKIP, plain, hhash, 1) - - TESTS(1); // * -// TESTS(2); // * -// TESTS(3); // * -// TESTS(5); // * -// TESTS(8); // * -// TESTS(9); -// TESTS(11); -// TESTS(13); // * - TESTS(15); -// TESTS(16); -// TESTS(21); // * -// TESTS(34); // * -// TESTS(55); // * -// TESTS(89); // * - - for (i = 1; i < argc; i++) { - if ((ret = read_whole_file(argv[i], - & buf, - & buf_len))) { - return 1; - } - - fprintf(stderr, "file %s is %zu bytes\n", - argv[i], buf_len); - - double min_time = -1.0; - double min_compression = 0.0; - - for (vector<test_result_base*>::iterator i = all_tests.begin(); - i != all_tests.end(); ++i) { - test_result_base *test = *i; - test->reset(); - - int iters = 100; - long start_test = get_millisecs_now(); - - do { - test->get(buf, buf_len, iters); - iters *= 3; - iters /= 2; - } while (get_millisecs_now() - start_test < 2000); - - test->stat(); - - if (min_time < 0.0) { - min_compression = test->compression(); - min_time = test->time(); - } - - if (min_time > test->time()) { - min_time = test->time(); - } - - if (min_compression > test->compression()) { - min_compression = test->compression(); - } - - test->print(); - } - -// for (vector<test_result_base*>::iterator i = all_tests.begin(); -// i != all_tests.end(); ++i) { -// test_result_base *test = *i; -// test->set_score(min_compression, min_time); -// } - -// sort(all_tests.begin(), all_tests.end(), compare_h()); - -// for (vector<test_result_base*>::iterator i = all_tests.begin(); -// i != all_tests.end(); ++i) { -// test_result_base *test = *i; -// test->print(); -// } - - free(buf); - buf = NULL; - } - - return 0; -} diff --git a/examples/compare_test.c b/examples/compare_test.c deleted file mode 100644 index f3b3ea2..0000000 --- a/examples/compare_test.c +++ /dev/null @@ -1,123 +0,0 @@ -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <string.h> -#include <assert.h> - -#include "xdelta3.h" - -#define NUM (1<<20) -#define ITERS 100 - -/* From wikipedia on RDTSC */ -inline uint64_t rdtsc() { - uint32_t lo, hi; - asm volatile ("rdtsc" : "=a" (lo), "=d" (hi)); - return (uint64_t)hi << 32 | lo; -} - -typedef int (*test_func)(const char *s1, const char *s2, int n); - -void run_test(const char *buf1, const char *buf2, - const char *name, test_func func) { - uint64_t start, end; - uint64_t accum = 0; - int i, x; - - for (i = 0; i < ITERS; i++) { - start = rdtsc(); - x = func(buf1, buf2, NUM); - end = rdtsc(); - accum += end - start; - assert(x == NUM - 1); - } - - accum /= ITERS; - - printf("%s : %qu cycles\n", name, accum); -} - -/* Build w/ -fno-builtin for this to be fast, this assumes that there - * is a difference at s1[n-1] */ -int memcmp_fake(const char *s1, const char *s2, int n) { - int x = memcmp(s1, s2, n); - return x < 0 ? n - 1 : n + 1; -} - -#define UNALIGNED_OK 1 -static inline int -test2(const char *s1c, const char *s2c, int n) -{ - int i = 0; -#if UNALIGNED_OK - int nint = n / sizeof(int); - - if (nint >> 3) - { - int j = 0; - const int *s1 = (const int*)s1c; - const int *s2 = (const int*)s2c; - int nint_8 = nint - 8; - - while (i <= nint_8 && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++]) { } - - i = (i - 1) * sizeof(int); - } -#endif - - while (i < n && s1c[i] == s2c[i]) - { - i++; - } - return i; -} - -static inline int -test1(const char *s1c, const char *s2c, int n) { - int i = 0; - while (i < n && s1c[i] == s2c[i]) - { - i++; - } - return i; -} - -int main(/*int argc, char **argv*/) { - char *buf1 = malloc(NUM+1); - char *buf2 = malloc(NUM+1); - int i; - - for (i = 0; i < NUM; i++) { - buf1[i] = buf2[i] = rand(); - } - - buf2[NUM-1]++; - - printf ("ALIGNED\n"); - - run_test(buf1, buf2, "memcmp", &memcmp_fake); - run_test(buf1, buf2, "test1", &test1); - run_test(buf1, buf2, "test2", &test2); - - for (i = 0; i < NUM; i++) { - buf1[i] = buf2[i+1] = rand(); - } - - buf2[NUM]++; - - printf ("UNALIGNED\n"); - - run_test(buf1, buf2+1, "memcmp", &memcmp_fake); - run_test(buf1, buf2+1, "test1", &test1); - run_test(buf1, buf2+1, "test2", &test2); - - return 0; -} diff --git a/examples/encode_decode_test.c b/examples/encode_decode_test.c deleted file mode 100644 index 7bcf109..0000000 --- a/examples/encode_decode_test.c +++ /dev/null @@ -1,204 +0,0 @@ -// -// Permission to distribute this example by -// Copyright (C) 2007 Ralf Junker -// Ralf Junker <delphi@yunqa.de> -// http://www.yunqa.de/delphi/ - -//--------------------------------------------------------------------------- - -#include <stdio.h> -#include <sys/stat.h> -#include "xdelta3.h" -#include "xdelta3.c" - -//--------------------------------------------------------------------------- - -int code ( - int encode, - FILE* InFile, - FILE* SrcFile , - FILE* OutFile, - int BufSize ) -{ - int r, ret; - struct stat statbuf; - xd3_stream stream; - xd3_config config; - xd3_source source; - void* Input_Buf; - int Input_Buf_Read; - - if (BufSize < XD3_ALLOCSIZE) - BufSize = XD3_ALLOCSIZE; - - memset (&stream, 0, sizeof (stream)); - memset (&source, 0, sizeof (source)); - - xd3_init_config(&config, XD3_ADLER32); - config.winsize = BufSize; - xd3_config_stream(&stream, &config); - - if (SrcFile) - { - r = fstat(fileno(SrcFile), &statbuf); - if (r) - return r; - source.size = statbuf.st_size; - source.blksize = BufSize; - source.curblk = malloc(source.blksize); - - /* Load 1st block of stream. */ - r = fseek(SrcFile, 0, SEEK_SET); - if (r) - return r; - source.onblk = fread((void*)source.curblk, 1, source.blksize, SrcFile); - source.curblkno = 0; - /* Set the stream. */ - xd3_set_source(&stream, &source); - } - - Input_Buf = malloc(BufSize); - - fseek(InFile, 0, SEEK_SET); - do - { - Input_Buf_Read = fread(Input_Buf, 1, BufSize, InFile); - if (Input_Buf_Read < BufSize) - { - xd3_set_flags(&stream, XD3_FLUSH | stream.flags); - } - xd3_avail_input(&stream, Input_Buf, Input_Buf_Read); - -process: - if (encode) - ret = xd3_encode_input(&stream); - else - ret = xd3_decode_input(&stream); - - switch (ret) - { - case XD3_INPUT: - { - fprintf (stderr,"XD3_INPUT\n"); - continue; - } - - case XD3_OUTPUT: - { - fprintf (stderr,"XD3_OUTPUT\n"); - r = fwrite(stream.next_out, 1, stream.avail_out, OutFile); - if (r != (int)stream.avail_out) - return r; - xd3_consume_output(&stream); - goto process; - } - - case XD3_GETSRCBLK: - { - fprintf (stderr,"XD3_GETSRCBLK %qd\n", source.getblkno); - if (SrcFile) - { - r = fseek(SrcFile, source.blksize * source.getblkno, SEEK_SET); - if (r) - return r; - source.onblk = fread((void*)source.curblk, 1, - source.blksize, SrcFile); - source.curblkno = source.getblkno; - } - goto process; - } - - case XD3_GOTHEADER: - { - fprintf (stderr,"XD3_GOTHEADER\n"); - goto process; - } - - case XD3_WINSTART: - { - fprintf (stderr,"XD3_WINSTART\n"); - goto process; - } - - case XD3_WINFINISH: - { - fprintf (stderr,"XD3_WINFINISH\n"); - goto process; - } - - default: - { - fprintf (stderr,"!!! INVALID %s %d !!!\n", - stream.msg, ret); - return ret; - } - - } - - } - while (Input_Buf_Read == BufSize); - - free(Input_Buf); - - free((void*)source.curblk); - xd3_close_stream(&stream); - xd3_free_stream(&stream); - - return 0; - -}; - - -int main(int argc, char* argv[]) -{ - FILE* InFile; - FILE* SrcFile; - FILE* OutFile; - int r; - - if (argc != 3) { - fprintf (stderr, "usage: %s source input output\n", argv[0]); - return 1; - } - - char *input = argv[2]; - char *source = argv[1]; - const char *output = "encoded.testdata"; - const char *decoded = "decoded.testdata"; - - /* Encode */ - - InFile = fopen(input, "rb"); - SrcFile = fopen(source, "rb"); - OutFile = fopen(output, "wb"); - - r = code (1, InFile, SrcFile, OutFile, 0x1000); - - fclose(OutFile); - fclose(SrcFile); - fclose(InFile); - - if (r) { - fprintf (stderr, "Encode error: %d\n", r); - return r; - } - - /* Decode */ - - InFile = fopen(output, "rb"); - SrcFile = fopen(source, "rb"); - OutFile = fopen(decoded, "wb"); - - r = code (0, InFile, SrcFile, OutFile, 0x1000); - - fclose(OutFile); - fclose(SrcFile); - fclose(InFile); - - if (r) { - fprintf (stderr, "Decode error: %d\n", r); - return r; - } - - return 0; -} diff --git a/examples/small_page_test.c b/examples/small_page_test.c deleted file mode 100755 index 2d9ae93..0000000 --- a/examples/small_page_test.c +++ /dev/null @@ -1,202 +0,0 @@ -/* Copyright (C) 2007 Josh MacDonald */ - -#include <stdio.h> - -#define PAGE_SIZE 4096 - -#define SPACE_MAX 131072 // how much memory per process -#define OUTPUT_MAX 1024 // max size for output -#define XD3_ALLOCSIZE 256 // internal size for various buffers -#define IOPT_SIZE 128 // instruction buffer - -// SPACE_MAX of 32K is sufficient for most inputs with XD3_COMPLEVEL_1 -// XD3_COMPLEVEL_9 requires about 4x more space than XD3_COMPLEVEL_1 - -#include "xdelta3.h" -#include "xdelta3.c" - -typedef struct _context { - uint8_t *buffer; - int allocated; -} context_t; - -static int max_allocated = 0; - -void* -process_alloc (void* opaque, usize_t items, usize_t size) -{ - context_t *ctx = (context_t*) opaque; - usize_t t = items * size; - void *ret; - - if (ctx->allocated + t > SPACE_MAX) - { - return NULL; - } - - ret = ctx->buffer + ctx->allocated; - ctx->allocated += t; - return ret; -} - -void -process_free (void* opaque, void *ptr) -{ -} - -int -process_page (int is_encode, - int (*func) (xd3_stream *), - const uint8_t *input, - usize_t input_size, - const uint8_t *source, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max, - int flags) { - - /* On my x86 this is 1072 of objects on the stack */ - xd3_stream stream; - xd3_config config; - xd3_source src; - context_t *ctx = calloc(SPACE_MAX, 1); - int ret; - - memset (&config, 0, sizeof(config)); - - if (ctx == NULL) - { - printf("calloc failed\n"); - return -1; - } - - ctx->buffer = (uint8_t*)ctx; - ctx->allocated = sizeof(*ctx); - - config.flags = flags; - config.winsize = PAGE_SIZE; - config.sprevsz = PAGE_SIZE; - config.srcwin_maxsz = PAGE_SIZE; - config.iopt_size = IOPT_SIZE; - config.alloc = &process_alloc; - config.freef = &process_free; - config.opaque = (void*) ctx; - - src.size = PAGE_SIZE; - src.blksize = PAGE_SIZE; - src.onblk = PAGE_SIZE; - src.curblk = source; - src.curblkno = 0; - - if ((ret = xd3_config_stream (&stream, &config)) != 0 || - (ret = xd3_set_source (&stream, &src)) != 0 || - (ret = xd3_process_stream (is_encode, - &stream, - func, 1, - input, input_size, - output, output_size, - output_size_max)) != 0) - { - if (stream.msg != NULL) - { - fprintf(stderr, "stream message: %s\n", stream.msg); - } - } - - xd3_free_stream (&stream); - if (max_allocated < ctx->allocated) - { - max_allocated = ctx->allocated; - fprintf(stderr, "max allocated %d\n", max_allocated); - } - - free(ctx); - return ret; -} - -int test(int stride, int encode_flags) -{ - uint8_t frompg[PAGE_SIZE]; - uint8_t topg[PAGE_SIZE]; - uint8_t output[OUTPUT_MAX]; - uint8_t reout[PAGE_SIZE]; - usize_t output_size; - usize_t re_size; - int i, j, ret; - - for (i = 0; i < PAGE_SIZE; i++) - { - topg[i] = frompg[i] = (rand() >> 3 ^ rand() >> 6 ^ rand() >> 9); - } - - // change 1 byte every stride - if (stride > 0) - { - for (j = stride; j <= PAGE_SIZE; j += stride) - { - topg[j - 1] ^= 0xff; - } - } - - if ((ret = process_page (1, xd3_encode_input, - topg, PAGE_SIZE, - frompg, output, - &output_size, OUTPUT_MAX, - encode_flags)) != 0) - { - fprintf (stderr, "encode failed: stride %u flags 0x%x\n", - stride, encode_flags); - return ret; - } - - if ((ret = process_page (0, xd3_decode_input, - output, output_size, - frompg, reout, - &re_size, PAGE_SIZE, - 0)) != 0) - { - fprintf (stderr, "decode failed: stride %u output_size %u flags 0x%x\n", - stride, output_size, encode_flags); - return ret; - } - - if (output_size > OUTPUT_MAX || re_size != PAGE_SIZE) - { - fprintf (stderr, "internal error: %u != %u\n", output_size, re_size); - return -1; - } - - for (i = 0; i < PAGE_SIZE; i++) - { - if (reout[i] != topg[i]) - { - fprintf (stderr, "encode-decode error: position %d\n", i); - return -1; - } - } - - fprintf(stderr, "stride %d flags 0x%x size %u ", - stride, encode_flags, output_size); - fprintf(stderr, "%s\n", (ret == 0) ? "OK" : "FAIL"); - - return 0; -} - -int main() -{ - int stride; - int level; - - for (level = 1; level < 10; level = (level == 1 ? 3 : level + 3)) - { - int lflag = level << XD3_COMPLEVEL_SHIFT; - - for (stride = 2; stride <= PAGE_SIZE; stride += 2) - { - test(stride, lflag); - test(stride, lflag | XD3_SEC_DJW); - } - } - - return 0; -} diff --git a/examples/speed_test.c b/examples/speed_test.c deleted file mode 100644 index d9ce5aa..0000000 --- a/examples/speed_test.c +++ /dev/null @@ -1,73 +0,0 @@ -/* Copyright (C) 2007 Josh MacDonald */ - -#include "test.h" - -usize_t bench_speed(const uint8_t *from_buf, const size_t from_len, - const uint8_t *to_buf, const size_t to_len, - uint8_t *delta_buf, const size_t delta_alloc, - int flags) { - usize_t delta_size; - int ret = xd3_encode_memory(to_buf, to_len, from_buf, from_len, - delta_buf, &delta_size, delta_alloc, flags); - if (ret != 0) { - fprintf(stderr, "encode failure: %d: %s\n", ret, xd3_strerror(ret)); - abort(); - } - return delta_size; -} - -int main(int argc, char **argv) { - int repeat, level; - char *from, *to; - uint8_t *from_buf = NULL, *to_buf = NULL, *delta_buf = NULL; - size_t from_len = 0, to_len, delta_alloc, delta_size = 0; - long start, finish; - int i, ret; - int flags; - - if (argc != 5) { - fprintf(stderr, "usage: speed_test LEVEL COUNT FROM TO\n"); - return 1; - } - - level = atoi(argv[1]); - repeat = atoi(argv[2]); - from = argv[3]; - to = argv[4]; - flags = (level << XD3_COMPLEVEL_SHIFT) & XD3_COMPLEVEL_MASK; - - if ((strcmp(from, "null") != 0 && - (ret = read_whole_file(from, &from_buf, &from_len))) || - (ret = read_whole_file(to, &to_buf, &to_len))) { - fprintf(stderr, "read_whole_file error\n"); - goto exit; - } - - delta_alloc = to_len * 11 / 10; - delta_buf = main_malloc(delta_alloc); - - start = get_millisecs_now(); - - for (i = 0; i < repeat; ++i) { - delta_size = bench_speed(from_buf, from_len, - to_buf, to_len, delta_buf, delta_alloc, flags); - } - - finish = get_millisecs_now(); - - fprintf(stderr, - "STAT: encode %3.2f ms from %s to %s repeat %d %zdbit delta %zd\n", - (double)(finish - start) / repeat, from, to, repeat, sizeof (xoff_t) * 8, delta_size); - - ret = 0; - - if (0) { - exit: - ret = 1; - } - - main_free(to_buf); - main_free(from_buf); - main_free(delta_buf); - return ret; -} diff --git a/examples/test.h b/examples/test.h deleted file mode 100644 index e8016bb..0000000 --- a/examples/test.h +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright (C) 2007 Josh MacDonald */ - -#define NOT_MAIN 1 - -#include "xdelta3.h" -#include "xdelta3.c" - -static int read_whole_file(const char *name, - uint8_t **buf_ptr, - size_t *buf_len) { - main_file file; - int ret; - xoff_t len; - usize_t nread; - main_file_init(&file); - file.filename = name; - ret = main_file_open(&file, name, XO_READ); - if (ret != 0) { - fprintf(stderr, "open failed\n"); - goto exit; - } - ret = main_file_stat(&file, &len, 0); - if (ret != 0) { - fprintf(stderr, "stat failed\n"); - goto exit; - } - - (*buf_len) = (size_t)len; - (*buf_ptr) = (uint8_t*) main_malloc(*buf_len); - ret = main_file_read(&file, *buf_ptr, *buf_len, &nread, - "read failed"); - if (ret == 0 && *buf_len == nread) { - ret = 0; - } else { - fprintf(stderr, "invalid read\n"); - ret = XD3_INTERNAL; - } - exit: - main_file_cleanup(&file); - return ret; -} - diff --git a/linkxd3lib.c b/linkxd3lib.c deleted file mode 100644 index 284cb0d..0000000 --- a/linkxd3lib.c +++ /dev/null @@ -1,46 +0,0 @@ -#include "xdelta3.h" - -extern int VVV; - -int VVV; - -void use(int r) -{ - VVV = r; -} - -int main() { - xd3_config config; - xd3_stream stream; - xd3_source source; - - xd3_init_config (& config, 0); - use (xd3_config_stream (&stream, &config)); - use (xd3_close_stream (&stream)); - xd3_abort_stream (&stream); - xd3_free_stream (&stream); - - xd3_avail_input (& stream, NULL, 0); - xd3_consume_output (& stream); - - use (xd3_bytes_on_srcblk (& source, 0)); - use (xd3_set_source (& stream, & source)); - xd3_set_flags (& stream, 0); - - use (xd3_decode_stream (& stream, NULL, 0, NULL, NULL, 0)); - use (xd3_decode_input (&stream)); - use (xd3_get_appheader (& stream, NULL, NULL)); - - use ((int) xd3_errstring (& stream)); - use ((int) xd3_strerror (0)); - -#if XD3_ENCODER - use (xd3_encode_input (&stream)); - use (xd3_encode_stream (& stream, NULL, 0, NULL, NULL, 0)); - use (xd3_set_appheader (& stream)); - use (xd3_encoder_used_source (& stream)); - use (xd3_encoder_srcbase (& stream)); - use (xd3_encoder_srclen (& stream)); -#endif - return 0; -} diff --git a/readme.txt b/readme.txt deleted file mode 100644 index be7c6ce..0000000 --- a/readme.txt +++ /dev/null @@ -1,34 +0,0 @@ -Xdelta 3.x readme.txt -Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007 -<josh.macdonald@gmail.com> - - -Thanks for downloading Xdelta! - -This directory contains the Xdelta3 command-line interface (CLI) and source -distribution for VCDIFF differential compression, a.k.a. delta -compression. The latest information and downloads are available here: - - http://xdelta.org/ - http://code.google.com/p/xdelta/ - -The command-line syntax: - - http://code.google.com/p/xdelta/wiki/CommandLineSyntax - -Run 'xdelta3 -h' for brief help. Run 'xdelta3 test' for built-in tests. - -Sample commands (like gzip, -e means encode, -d means decode) - - xdelta3 -9 -S djw -e -vfs OLD_FILE NEW_FILE DELTA_FILE - xdelta3 -d -vfs OLD_FILE DELTA_FILE DECODED_FILE - -File bug reports and browse open support issues here: - - http://code.google.com/p/xdelta/issues/list - -The source distribution contains the C/C++/Python APIs, Unix, Microsoft VC++ -and Cygwin builds. Xdelta3 is covered under the terms of the GPL, see -COPYING. - -Commercial inquiries welcome, please contact <josh.macdonald@gmail.com> diff --git a/setup.py b/setup.py deleted file mode 100644 index 0bb39e1..0000000 --- a/setup.py +++ /dev/null @@ -1,59 +0,0 @@ -# xdelta 3 - delta compression tools and library -# Copyright (C) 2004, 2007. Joshua P. MacDonald -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# -from distutils.core import setup, Extension -from distutils.util import get_platform - -# External compression support works on Windows/Cygwin, but not from -# within the Python module. It's something to do with fork() and -# exec() support. -#platform = get_platform() -#is_cygwin = platform.startswith('cygwin') - -xdelta3_ext = Extension('xdelta3main', - ['xdelta3.c'], - define_macros = [ - ('PYTHON_MODULE',1), - ('SECONDARY_DJW',1), - ('VCDIFF_TOOLS',1), - ('GENERIC_ENCODE_TABLES',0), - ('XD3_POSIX',1), - ('XD3_USE_LARGEFILE64',0), - - # the fork/exec stuff doesn't - # work inside python. - ('EXTERNAL_COMPRESSION',0), - - ('REGRESSION_TEST',0), - ('SECONDARY_FGK',0), - ('XD3_DEBUG',0), - ], - extra_compile_args = [ '-O3', - '-g', - '-fno-builtin', - # '-arch', 'x86_64', - ]) - -# $Format: "REL='$Xdelta3Version$'" $ -REL='3.0u' - -# This provides xdelta3.main(), which calls the xdelta3 command-line main() -# from python. -setup(name='xdelta3main', - version=REL, - ext_modules=[xdelta3_ext]) diff --git a/testing/Makefile b/testing/Makefile deleted file mode 100755 index 281ef11..0000000 --- a/testing/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -CFLAGS = -g -Wall -I.. -DXD3_DEBUG=1 -#CFLAGS = -g -Wall -I.. -DXD3_DEBUG=2 -#CFLAGS = -O2 -Wall -I.. -DXD3_DEBUG=0 -DNDEBUG=1 - -DEPS = ../*.h ../*.c *.cc *.h - -TARGETS = xdelta3-regtest - -all: $(TARGETS) - -xdelta3-regtest: $(DEPS) - $(CXX) $(CFLAGS) regtest.cc -o xdelta3-regtest - -clean: - rm -f *.exe *.stackdump $(TARGETS) diff --git a/testing/cmp.h b/testing/cmp.h deleted file mode 100644 index d96c386..0000000 --- a/testing/cmp.h +++ /dev/null @@ -1,64 +0,0 @@ -/* -*- Mode: C++ -*- */ -namespace regtest { - -inline size_t CmpDifferentBlockBytes(const Block &a, const Block &b) { - size_t total = 0; - size_t i = 0; - size_t m = min(a.Size(), b.Size()); - - for (; i < m; i++) { - if (a[i] != b[i]) { - total++; - } - } - - total += a.Size() - i; - total += b.Size() - i; - - return total; -} - -inline xoff_t CmpDifferentBytes(const FileSpec &a, const FileSpec &b) { - Block block_a, block_b; - xoff_t total = 0; - FileSpec::iterator a_i(a), b_i(b); - - for (; !a_i.Done() && !b_i.Done(); a_i.Next(), b_i.Next()) { - - a_i.Get(&block_a); - b_i.Get(&block_b); - - total += CmpDifferentBlockBytes(block_a, block_b); - } - - for (; !a_i.Done(); a_i.Next()) { - total += a_i.BytesOnBlock(); - } - for (; !b_i.Done(); b_i.Next()) { - total += b_i.BytesOnBlock(); - } - - return total; -} - -inline bool ExtFile::EqualsSpec(const FileSpec &spec) const { - main_file t; - main_file_init(&t); - CHECK_EQ(0, main_file_open(&t, Name(), XO_READ)); - - Block tblock; - Block sblock; - for (BlockIterator iter(spec); !iter.Done(); iter.Next()) { - iter.Get(&sblock); - tblock.SetSize(sblock.Size()); - usize_t tread; - CHECK_EQ(0, main_file_read(&t, tblock.Data(), tblock.Size(), &tread, "read failed")); - CHECK_EQ(0, CmpDifferentBlockBytes(tblock, sblock)); - } - - CHECK_EQ(0, main_file_close(&t)); - main_file_cleanup(&t); - return true; -} - -} // namespace regtest diff --git a/testing/delta.h b/testing/delta.h deleted file mode 100644 index 58fbaac..0000000 --- a/testing/delta.h +++ /dev/null @@ -1,79 +0,0 @@ -// Mode: -*- C++ -*- - -namespace regtest { - -class Delta { -public: - Delta(const Block &block); - - ~Delta() { - xd3_free_stream(&stream_); - } - - xoff_t AddedBytes() const { - return stream_.whole_target.addslen; - } - - xoff_t Windows() const { - return stream_.whole_target.wininfolen; - } - - void Print() const; - -private: - xd3_stream stream_; -}; - -Delta::Delta(const Block &block) { - int ret; - xd3_config config; - memset(&stream_, 0, sizeof (stream_)); - memset(&config, 0, sizeof (config)); - - xd3_init_config(&config, XD3_SKIP_EMIT | XD3_ADLER32_NOVER); - - CHECK_EQ(0, xd3_config_stream (&stream_, &config)); - - xd3_avail_input (&stream_, block.Data(), block.Size()); - - bool done = false; - while (!done) { - ret = xd3_decode_input(&stream_); - - switch (ret) { - case XD3_INPUT: - done = true; - break; - case XD3_OUTPUT: - CHECK_EQ(0, xd3_whole_append_window (&stream_)); - break; - case XD3_GOTHEADER: - case XD3_WINSTART: - case XD3_WINFINISH: - break; - default: - DP(RINT "error code %s\n", xd3_strerror (ret)); - abort(); - } - } -} - -void Delta::Print() const { - for (size_t i = 0; i < stream_.whole_target.instlen; i++) { - xd3_winst &winst = stream_.whole_target.inst[i]; - switch (winst.type) { - case XD3_RUN: - DP(RINT "%"Q"u run %u\n", winst.position, winst.size); - break; - case XD3_ADD: - DP(RINT "%"Q"u add %u\n", winst.position, winst.size); - break; - default: - DP(RINT "%"Q"u copy %u @ %"Q"u (mode %u)\n", - winst.position, winst.size, winst.addr, winst.mode); - break; - } - } -} - -} // namespace diff --git a/testing/file.h b/testing/file.h deleted file mode 100644 index 30a8428..0000000 --- a/testing/file.h +++ /dev/null @@ -1,367 +0,0 @@ -/* -*- Mode: C++ -*- */ -namespace regtest { - -class Block; -class BlockIterator; -class TmpFile; - -class FileSpec { - public: - FileSpec(MTRandom *rand) - : rand_(rand) { - } - - // Generates a file with a known size - void GenerateFixedSize(xoff_t size) { - Reset(); - - for (xoff_t p = 0; p < size; ) { - xoff_t t = min(Constants::BLOCK_SIZE, size - p); - table_.insert(make_pair(p, Segment(t, rand_))); - p += t; - } - } - - // Generates a file with exponential-random distributed size - void GenerateRandomSize(xoff_t mean) { - GenerateFixedSize(rand_->ExpRand(mean)); - } - - // Returns the size of the file - xoff_t Size() const { - if (table_.empty()) { - return 0; - } - SegmentMap::const_iterator i = --table_.end(); - return i->first + i->second.Size(); - } - - // Returns the number of blocks - xoff_t Blocks(size_t blksize = Constants::BLOCK_SIZE) const { - if (table_.empty()) { - return 0; - } - return ((Size() - 1) / blksize) + 1; - } - - // Returns the number of segments - xoff_t Segments() const { - return table_.size(); - } - - // Create a mutation according to "what". - void ModifyTo(const Mutator &mutator, - FileSpec *modify) const { - modify->Reset(); - mutator.Mutate(&modify->table_, &table_, rand_); - modify->CheckSegments(); - } - - void CheckSegments() const { - for (SegmentMap::const_iterator iter(table_.begin()); - iter != table_.end(); ) { - SegmentMap::const_iterator iter0(iter++); - if (iter == table_.end()) { - break; - } - CHECK_EQ(iter0->first + iter0->second.Size(), iter->first); - } - } - - void Reset() { - table_.clear(); - } - - void Print() const { - for (SegmentMap::const_iterator iter(table_.begin()); - iter != table_.end(); - ++iter) { - const Segment &seg = iter->second; - cerr << "Segment at " << iter->first << " (" << seg << ")" << endl; - } - } - - void PrintData() const; - - void WriteTmpFile(TmpFile *f) const; - - typedef BlockIterator iterator; - - private: - friend class BlockIterator; - - MTRandom *rand_; - SegmentMap table_; -}; - -class Block { -public: - Block() - : data_(NULL), - data_size_(0), - size_(0) { } - - ~Block() { - if (data_) { - delete [] data_; - } - } - - size_t Size() const { - return size_; - } - - uint8_t operator[](size_t i) const { - CHECK_LT(i, size_); - return data_[i]; - } - - uint8_t* Data() const { - if (data_ == NULL) { - CHECK_EQ(0, size_); - data_size_ = 1; - data_ = new uint8_t[1]; - } - return data_; - } - - // For writing to blocks - void Append(const uint8_t *data, size_t size); - - // For cleaing a block - void Reset() { - size_ = 0; - } - - void Print() const; - - void WriteTmpFile(TmpFile *f) const; - - void SetSize(size_t size) { - size_ = size; - - if (data_size_ < size) { - if (data_) { - delete [] data_; - } - data_ = new uint8_t[size]; - data_size_ = size; - } - } -private: - friend class BlockIterator; - - mutable uint8_t *data_; - mutable size_t data_size_; - size_t size_; -}; - -class BlockIterator { -public: - explicit BlockIterator(const FileSpec& spec) - : spec_(spec), - blkno_(0), - blksize_(Constants::BLOCK_SIZE) { } - - BlockIterator(const FileSpec& spec, - size_t blksize) - : spec_(spec), - blkno_(0), - blksize_(blksize) { } - - bool Done() const { - return blkno_ >= spec_.Blocks(blksize_); - } - - void Next() { - blkno_++; - } - - xoff_t Blkno() const { - return blkno_; - } - - xoff_t Offset() const { - return blkno_ * blksize_; - } - - void SetBlock(xoff_t blkno) { - blkno_ = blkno; - } - - void Get(Block *block) const; - - size_t BytesOnBlock() const { - xoff_t blocks = spec_.Blocks(blksize_); - xoff_t size = spec_.Size(); - - CHECK((blkno_ < blocks) || - (blkno_ == blocks && size % blksize_ == 0)); - - if (blkno_ == blocks) { - return 0; - } - if (blkno_ + 1 == blocks) { - return ((size - 1) % blksize_) + 1; - } - return blksize_; - } - - size_t BlockSize() const { - return blksize_; - } - -private: - const FileSpec& spec_; - xoff_t blkno_; - size_t blksize_; -}; - -class ExtFile { -public: - ExtFile() { - static int static_counter = 0; - char buf[32]; - snprintf(buf, 32, "/tmp/regtest.%d", static_counter++); - filename_.append(buf); - unlink(filename_.c_str()); - } - - ~ExtFile() { - unlink(filename_.c_str()); - } - - const char* Name() const { - return filename_.c_str(); - } - - // Check whether a real file matches a file spec. - bool EqualsSpec(const FileSpec &spec) const; - -protected: - string filename_; -}; - -class TmpFile : public ExtFile { -public: - // TODO this is a little unportable! - TmpFile() { - main_file_init(&file_); - CHECK_EQ(0, main_file_open(&file_, filename_.c_str(), XO_WRITE)); - } - - ~TmpFile() { - main_file_cleanup(&file_); - } - - void Append(const Block *block) { - CHECK_EQ(0, main_file_write(&file_, - block->Data(), block->Size(), - "tmpfile write failed")); - } - - - const char* Name() const { - if (main_file_isopen(&file_)) { - CHECK_EQ(0, main_file_close(&file_)); - } - return ExtFile::Name(); - } - -private: - mutable main_file file_; -}; - -inline void BlockIterator::Get(Block *block) const { - xoff_t offset = blkno_ * blksize_; - const SegmentMap &table = spec_.table_; - size_t got = 0; - block->SetSize(BytesOnBlock()); - - SegmentMap::const_iterator pos = table.upper_bound(offset); - if (pos == table.begin()) { - CHECK_EQ(0, spec_.Size()); - return; - } - --pos; - - while (got < block->size_) { - CHECK(pos != table.end()); - CHECK_GE(offset, pos->first); - - const Segment &seg = pos->second; - - // The position of this segment may start before this block starts, - // and then the position of the data may be offset from the seeding - // position. - size_t seg_offset = offset - pos->first; - size_t advance = min(seg.Size() - seg_offset, - blksize_ - got); - - seg.Fill(seg_offset, advance, block->data_ + got); - - got += advance; - offset += advance; - ++pos; - } -} - -inline void Block::Append(const uint8_t *data, size_t size) { - if (data_ == NULL) { - CHECK_EQ(0, size_); - CHECK_EQ(0, data_size_); - data_ = new uint8_t[Constants::BLOCK_SIZE]; - data_size_ = Constants::BLOCK_SIZE; - } - - if (size_ + size > data_size_) { - uint8_t *tmp = data_; - while (size_ + size > data_size_) { - data_size_ *= 2; - } - data_ = new uint8_t[data_size_]; - memcpy(data_, tmp, size_); - delete tmp; - } - - memcpy(data_ + size_, data, size); - size_ += size; -} - -inline void FileSpec::PrintData() const { - Block block; - for (BlockIterator iter(*this); !iter.Done(); iter.Next()) { - iter.Get(&block); - block.Print(); - } -} - -inline void Block::Print() const { - xoff_t pos = 0; - for (size_t i = 0; i < Size(); i++) { - if (pos % 16 == 0) { - DP(RINT "%5"Q"x: ", pos); - } - DP(RINT "%02x ", (*this)[i]); - if (pos % 16 == 15) { - DP(RINT "\n"); - } - pos++; - } - DP(RINT "\n"); -} - -inline void FileSpec::WriteTmpFile(TmpFile *f) const { - Block block; - for (BlockIterator iter(*this); !iter.Done(); iter.Next()) { - iter.Get(&block); - f->Append(&block); - } -} - -inline void Block::WriteTmpFile(TmpFile *f) const { - f->Append(this); -} - -} // namespace regtest - diff --git a/testing/modify.h b/testing/modify.h deleted file mode 100644 index 67cccd9..0000000 --- a/testing/modify.h +++ /dev/null @@ -1,421 +0,0 @@ -// -*- Mode: C++ -*- -namespace regtest { - -class Mutator { -public: - virtual ~Mutator() { } - virtual void Mutate(SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) const = 0; -}; - -class Change { -public: - enum Kind { - MODIFY = 1, - ADD = 2, - DELETE = 3, - MOVE = 4, - COPY = 5, - OVERWRITE = 6, - }; - - // Constructor for modify, add, delete. - Change(Kind kind, xoff_t size, xoff_t addr1) - : kind(kind), - size(size), - addr1(addr1), - insert(NULL) { - CHECK(kind != MOVE && kind != COPY && kind != OVERWRITE); - } - - // Constructor for modify, add w/ provided data. - Change(Kind kind, xoff_t size, xoff_t addr1, Segment *insert) - : kind(kind), - size(size), - addr1(addr1), - insert(insert) { - CHECK(kind != MOVE && kind != COPY && kind != OVERWRITE); - } - - // Constructor for move - Change(Kind kind, xoff_t size, xoff_t addr1, xoff_t addr2) - : kind(kind), - size(size), - addr1(addr1), - addr2(addr2), - insert(NULL) { - CHECK(kind == MOVE || kind == COPY || kind == OVERWRITE); - } - - Kind kind; - xoff_t size; - xoff_t addr1; - xoff_t addr2; - Segment *insert; // For modify and/or add -}; - -typedef list<Change> ChangeList; - -class ChangeListMutator : public Mutator { -public: - ChangeListMutator(const ChangeList &cl) - : cl_(cl) { } - - ChangeListMutator() { } - - void Mutate(SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) const; - - static void Mutate(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand); - - static void AddChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand); - - static void ModifyChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand); - - static void DeleteChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand); - - static void MoveChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand); - - static void OverwriteChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand); - - static void CopyChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand); - - static void AppendCopy(SegmentMap *table, - const SegmentMap *source_table, - xoff_t copy_offset, - xoff_t append_offset, - xoff_t length); - - ChangeList* Changes() { - return &cl_; - } - - const ChangeList* Changes() const { - return &cl_; - } - -private: - ChangeList cl_; -}; - -void ChangeListMutator::Mutate(SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) const { - // The speed of processing gigabytes of data is so slow compared with - // these table-copy operations, no attempt to make this fast. - SegmentMap tmp; - - for (ChangeList::const_iterator iter(cl_.begin()); iter != cl_.end(); ++iter) { - const Change &ch = *iter; - tmp.clear(); - Mutate(ch, &tmp, source_table, rand); - tmp.swap(*table); - source_table = table; - } -} - -void ChangeListMutator::Mutate(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) { - switch (ch.kind) { - case Change::ADD: - AddChange(ch, table, source_table, rand); - break; - case Change::MODIFY: - ModifyChange(ch, table, source_table, rand); - break; - case Change::DELETE: - DeleteChange(ch, table, source_table, rand); - break; - case Change::COPY: - CopyChange(ch, table, source_table, rand); - break; - case Change::MOVE: - MoveChange(ch, table, source_table, rand); - break; - case Change::OVERWRITE: - OverwriteChange(ch, table, source_table, rand); - break; - } -} - -void ChangeListMutator::ModifyChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) { - xoff_t m_start = ch.addr1; - xoff_t m_end = m_start + ch.size; - xoff_t i_start = 0; - xoff_t i_end = 0; - - for (SegmentMap::const_iterator iter(source_table->begin()); - iter != source_table->end(); - ++iter) { - const Segment &seg = iter->second; - i_start = iter->first; - i_end = i_start + seg.Size(); - - if (i_end <= m_start || i_start >= m_end) { - table->insert(table->end(), make_pair(i_start, seg)); - continue; - } - - if (i_start < m_start) { - table->insert(table->end(), - make_pair(i_start, - seg.Subseg(0, m_start - i_start))); - } - - // Insert the entire segment, even though it may extend into later - // segments. This condition avoids inserting it during later - // segments. - if (m_start >= i_start) { - if (ch.insert != NULL) { - table->insert(table->end(), make_pair(m_start, *ch.insert)); - } else { - Segment part(m_end - m_start, rand); - table->insert(table->end(), make_pair(m_start, part)); - } - } - - if (i_end > m_end) { - table->insert(table->end(), - make_pair(m_end, - seg.Subseg(m_end - i_start, i_end - m_end))); - } - } - - CHECK_LE(m_end, i_end); -} - -void ChangeListMutator::AddChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) { - xoff_t m_start = ch.addr1; - xoff_t i_start = 0; - xoff_t i_end = 0; - - for (SegmentMap::const_iterator iter(source_table->begin()); - iter != source_table->end(); - ++iter) { - const Segment &seg = iter->second; - i_start = iter->first; - i_end = i_start + seg.Size(); - - if (i_end <= m_start) { - table->insert(table->end(), make_pair(i_start, seg)); - continue; - } - - if (i_start > m_start) { - table->insert(table->end(), make_pair(i_start + ch.size, seg)); - continue; - } - - if (i_start < m_start) { - table->insert(table->end(), - make_pair(i_start, - seg.Subseg(0, m_start - i_start))); - } - - if (ch.insert != NULL) { - table->insert(table->end(), make_pair(m_start, *ch.insert)); - } else { - Segment addseg(ch.size, rand); - table->insert(table->end(), make_pair(m_start, addseg)); - } - - if (m_start < i_end) { - table->insert(table->end(), - make_pair(m_start + ch.size, - seg.Subseg(m_start - i_start, i_end - m_start))); - } - } - - CHECK_LE(m_start, i_end); - - // Special case for add at end-of-input. - if (m_start == i_end) { - Segment addseg(ch.size, rand); - table->insert(table->end(), make_pair(m_start, addseg)); - } -} - -void ChangeListMutator::DeleteChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) { - xoff_t m_start = ch.addr1; - xoff_t m_end = m_start + ch.size; - xoff_t i_start = 0; - xoff_t i_end = 0; - - for (SegmentMap::const_iterator iter(source_table->begin()); - iter != source_table->end(); - ++iter) { - const Segment &seg = iter->second; - i_start = iter->first; - i_end = i_start + seg.Size(); - - if (i_end <= m_start) { - table->insert(table->end(), make_pair(i_start, seg)); - continue; - } - - if (i_start >= m_end) { - table->insert(table->end(), make_pair(i_start - ch.size, seg)); - continue; - } - - if (i_start < m_start) { - table->insert(table->end(), - make_pair(i_start, - seg.Subseg(0, m_start - i_start))); - } - - if (i_end > m_end) { - table->insert(table->end(), - make_pair(m_end - ch.size, - seg.Subseg(m_end - i_start, i_end - m_end))); - } - } - - CHECK_LT(m_start, i_end); - CHECK_LE(m_end, i_end); -} - -void ChangeListMutator::MoveChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) { - SegmentMap tmp; - CHECK_NE(ch.addr1, ch.addr2); - CopyChange(ch, &tmp, source_table, rand); - Change d(Change::DELETE, ch.size, - ch.addr1 < ch.addr2 ? ch.addr1 : ch.addr1 + ch.size); - DeleteChange(d, table, &tmp, rand); -} - -void ChangeListMutator::OverwriteChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) { - SegmentMap tmp; - CHECK_NE(ch.addr1, ch.addr2); - CopyChange(ch, &tmp, source_table, rand); - Change d(Change::DELETE, ch.size, ch.addr2 + ch.size); - DeleteChange(d, table, &tmp, rand); -} - -void ChangeListMutator::CopyChange(const Change &ch, - SegmentMap *table, - const SegmentMap *source_table, - MTRandom *ignore) { - xoff_t m_start = ch.addr2; - xoff_t c_start = ch.addr1; - xoff_t i_start = 0; - xoff_t i_end = 0; - - // Like AddChange() with AppendCopy instead of a random segment. - for (SegmentMap::const_iterator iter(source_table->begin()); - iter != source_table->end(); - ++iter) { - const Segment &seg = iter->second; - i_start = iter->first; - i_end = i_start + seg.Size(); - - if (i_end <= m_start) { - table->insert(table->end(), make_pair(i_start, seg)); - continue; - } - - if (i_start > m_start) { - table->insert(table->end(), make_pair(i_start + ch.size, seg)); - continue; - } - - if (i_start < m_start) { - table->insert(table->end(), - make_pair(i_start, - seg.Subseg(0, m_start - i_start))); - } - - AppendCopy(table, source_table, c_start, m_start, ch.size); - - if (m_start < i_end) { - table->insert(table->end(), - make_pair(m_start + ch.size, - seg.Subseg(m_start - i_start, i_end - m_start))); - } - } - - CHECK_LE(m_start, i_end); - - // Special case for copy to end-of-input. - if (m_start == i_end) { - AppendCopy(table, source_table, c_start, m_start, ch.size); - } -} - -void ChangeListMutator::AppendCopy(SegmentMap *table, - const SegmentMap *source_table, - xoff_t copy_offset, - xoff_t append_offset, - xoff_t length) { - SegmentMap::const_iterator pos(source_table->upper_bound(copy_offset)); - --pos; - xoff_t got = 0; - - while (got < length) { - size_t seg_offset = copy_offset - pos->first; - size_t advance = min(pos->second.Size() - seg_offset, - (size_t)(length - got)); - - table->insert(table->end(), - make_pair(append_offset, - pos->second.Subseg(seg_offset, - advance))); - - got += advance; - copy_offset += advance; - append_offset += advance; - ++pos; - } -} - -class Modify1stByte : public Mutator { -public: - void Mutate(SegmentMap *table, - const SegmentMap *source_table, - MTRandom *rand) const { - ChangeListMutator::Mutate(Change(Change::MODIFY, 1, 0), - table, source_table, rand); - } -}; - -} // namespace regtest diff --git a/testing/random.h b/testing/random.h deleted file mode 100644 index f2cb167..0000000 --- a/testing/random.h +++ /dev/null @@ -1,140 +0,0 @@ -/* -*- Mode: C++ -*- */ -/* This is public-domain Mersenne Twister code, - * attributed to Michael Brundage. Thanks! - * http://www.qbrundage.com/michaelb/pubs/essays/random_number_generation.html - */ -#include <math.h> - -namespace regtest { - -class MTRandom { - public: - static const uint32_t TEST_SEED1 = 5489UL; - - static const int MT_LEN = 624; - static const int MT_IA = 397; - static const uint32_t UPPER_MASK = 0x80000000; - static const uint32_t LOWER_MASK = 0x7FFFFFFF; - static const uint32_t MATRIX_A = 0x9908B0DF; - - MTRandom() { - Init(TEST_SEED1); - } - - MTRandom(uint32_t seed) { - Init(seed); - } - - uint32_t Rand32 () { - uint32_t y; - static unsigned long mag01[2] = { - 0 , MATRIX_A - }; - - if (mt_index_ >= MT_LEN) { - int kk; - - for (kk = 0; kk < MT_LEN - MT_IA; kk++) { - y = (mt_buffer_[kk] & UPPER_MASK) | (mt_buffer_[kk + 1] & LOWER_MASK); - mt_buffer_[kk] = mt_buffer_[kk + MT_IA] ^ (y >> 1) ^ mag01[y & 0x1UL]; - } - for (;kk < MT_LEN - 1; kk++) { - y = (mt_buffer_[kk] & UPPER_MASK) | (mt_buffer_[kk + 1] & LOWER_MASK); - mt_buffer_[kk] = mt_buffer_[kk + (MT_IA - MT_LEN)] ^ (y >> 1) ^ mag01[y & 0x1UL]; - } - y = (mt_buffer_[MT_LEN - 1] & UPPER_MASK) | (mt_buffer_[0] & LOWER_MASK); - mt_buffer_[MT_LEN - 1] = mt_buffer_[MT_IA - 1] ^ (y >> 1) ^ mag01[y & 0x1UL]; - - mt_index_ = 0; - } - - y = mt_buffer_[mt_index_++]; - - y ^= (y >> 11); - y ^= (y << 7) & 0x9d2c5680UL; - y ^= (y << 15) & 0xefc60000UL; - y ^= (y >> 18); - - return y; - } - - uint32_t ExpRand32(uint32_t mean) { - double mean_d = mean; - double erand = log (1.0 / (Rand32() / (double)UINT32_MAX)); - uint32_t x = (uint32_t) (mean_d * erand + 0.5); - return x; - } - - uint64_t Rand64() { - return ((uint64_t)Rand32() << 32) | Rand32(); - } - - uint64_t ExpRand64(uint64_t mean) { - double mean_d = mean; - double erand = log (1.0 / (Rand64() / (double)UINT32_MAX)); - uint64_t x = (uint64_t) (mean_d * erand + 0.5); - return x; - } - - template <typename T> - T Rand() { - switch (sizeof(T)) { - case sizeof(uint32_t): - return Rand32(); - case sizeof(uint64_t): - return Rand64(); - default: - cerr << "Invalid sizeof T" << endl; - abort(); - } - } - - template <typename T> - T ExpRand(T mean) { - switch (sizeof(T)) { - case sizeof(uint32_t): - return ExpRand32(mean); - case sizeof(uint64_t): - return ExpRand64(mean); - default: - cerr << "Invalid sizeof T" << endl; - abort(); - } - } - - private: - void Init(uint32_t seed) { - mt_buffer_[0] = seed; - mt_index_ = MT_LEN; - for (int i = 1; i < MT_LEN; i++) { - /* See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. */ - /* In the previous versions, MSBs of the seed affect */ - /* only MSBs of the array mt[]. */ - /* 2002/01/09 modified by Makoto Matsumoto */ - mt_buffer_[i] = - (1812433253UL * (mt_buffer_[i-1] ^ (mt_buffer_[i-1] >> 30)) + i); - } - } - - int mt_index_; - uint32_t mt_buffer_[MT_LEN]; -}; - -class MTRandom8 { -public: - MTRandom8(MTRandom *rand) - : rand_(rand) { - } - - uint8_t Rand8() { - uint32_t r = rand_->Rand32(); - - // TODO: make this use a single byte at a time? - return (r & 0xff) ^ (r >> 7) ^ (r >> 15) ^ (r >> 21); - } - -private: - MTRandom *rand_; -}; - -} // namespace regtest diff --git a/testing/segment.h b/testing/segment.h deleted file mode 100644 index 1dabf5c..0000000 --- a/testing/segment.h +++ /dev/null @@ -1,100 +0,0 @@ -// -*- Mode: C++ -*- - -namespace regtest { - -class Segment { - public: - Segment(size_t size, MTRandom *rand) - : size_(size), - seed_(rand->Rand32()), - seed_offset_(0), - data_(NULL) { - CHECK_GT(size_, 0); - } - - Segment(size_t size, uint32_t seed) - : size_(size), - seed_(seed), - seed_offset_(0), - data_(NULL) { - CHECK_GT(size_, 0); - } - - Segment(size_t size, uint8_t *data) - : size_(size), - seed_(0), - seed_offset_(0), - data_(data) { - CHECK_GT(size_, 0); - } - - size_t Size() const { - return size_; - } - - Segment Subseg(size_t start, size_t size) const { - CHECK_LE(start + size, size_); - if (data_) { - return Segment(size, data_ + start); - } else { - return Segment(size, seed_, seed_offset_ + start); - } - } - - void Fill(size_t seg_offset, size_t size, uint8_t *data) const { - CHECK_LE(seg_offset + size, size_); - if (data_) { - memcpy(data, data_ + seg_offset, size); - } else { - size_t skip = seg_offset + seed_offset_; - MTRandom gen(seed_); - MTRandom8 gen8(&gen); - while (skip--) { - gen8.Rand8(); - } - for (size_t i = 0; i < size; i++) { - data[i] = gen8.Rand8(); - } - } - } - -private: - // Used by Subseg() - Segment(size_t size, uint32_t seed, size_t seed_offset) - : size_(size), - seed_(seed), - seed_offset_(seed_offset), - data_(NULL) { - CHECK_GT(size_, 0); - } - - friend ostream& operator<<(ostream& os, const Segment &seg); - - size_t size_; // Size of this segment - - // For random segments - uint32_t seed_; // Seed used for generating byte sequence - size_t seed_offset_; // Seed positions the sequence this many bytes - // before its beginning. - - // For literal segments (data is not owned) - uint8_t *data_; -}; - -ostream& operator<<(ostream& os, const Segment &seg) { - if (seg.data_) { - for (size_t i = 0; i < seg.size_; i++) { - char buf[10]; - sprintf(buf, "%02x ", seg.data_[i]); - os << buf; - } - return os; - } else { - return os << "size=" << seg.size_ << ",seed=" << seg.seed_ - << ",skip=" << seg.seed_offset_; - } -} - -typedef map<xoff_t, Segment> SegmentMap; - -} // namespace regtest diff --git a/testing/sizes.h b/testing/sizes.h deleted file mode 100644 index 6b70892..0000000 --- a/testing/sizes.h +++ /dev/null @@ -1,69 +0,0 @@ -// -*- Mode: C++ -*- -namespace regtest { - -template <typename T, typename U> -class SizeIterator { - public: - SizeIterator(MTRandom *rand, size_t howmany) - : rand_(rand), - count_(0), - fixed_(U::sizes), - fixed_size_(SIZEOF_ARRAY(U::sizes)), - howmany_(howmany) { } - - T Get() { - if (count_ < fixed_size_) { - return fixed_[count_]; - } - return rand_->Rand<T>() % U::max_value; - } - - bool Done() { - return count_ >= fixed_size_ && count_ >= howmany_; - } - - void Next() { - count_++; - } - - private: - MTRandom *rand_; - size_t count_; - T* fixed_; - size_t fixed_size_; - size_t howmany_; -}; - -class SmallSizes { -public: - static size_t sizes[]; - static size_t max_value; -}; - -size_t SmallSizes::sizes[] = { - 0, 1, Constants::BLOCK_SIZE / 4, 3333, - Constants::BLOCK_SIZE - (Constants::BLOCK_SIZE / 3), - Constants::BLOCK_SIZE, - Constants::BLOCK_SIZE + (Constants::BLOCK_SIZE / 3), - 2 * Constants::BLOCK_SIZE - (Constants::BLOCK_SIZE / 3), - 2 * Constants::BLOCK_SIZE, - 2 * Constants::BLOCK_SIZE + (Constants::BLOCK_SIZE / 3), -}; - -size_t SmallSizes::max_value = Constants::BLOCK_SIZE * 3; - -class LargeSizes { -public: - static size_t sizes[]; - static size_t max_value; -}; - -size_t LargeSizes::sizes[] = { - 1 << 20, - 1 << 18, - 1 << 16, -}; - -size_t LargeSizes::max_value = 1<<20; - -} // namespace regtest diff --git a/testing/test.h b/testing/test.h deleted file mode 100644 index f2b46f3..0000000 --- a/testing/test.h +++ /dev/null @@ -1,110 +0,0 @@ -// -*- Mode: C++ -*- - -extern "C" { -#define NOT_MAIN 1 -#define REGRESSION_TEST 0 -#define VCDIFF_TOOLS 1 -#include "../xdelta3.c" -} - -#define CHECK_EQ(x,y) CHECK_OP(x,y,==) -#define CHECK_NE(x,y) CHECK_OP(x,y,!=) -#define CHECK_LT(x,y) CHECK_OP(x,y,<) -#define CHECK_GT(x,y) CHECK_OP(x,y,>) -#define CHECK_LE(x,y) CHECK_OP(x,y,<=) -#define CHECK_GE(x,y) CHECK_OP(x,y,>=) - -#define CHECK_OP(x,y,OP) \ - do { \ - typeof(x) _x(x); \ - typeof(x) _y(y); \ - if (!(_x OP _y)) { \ - cerr << __FILE__ << ":" << __LINE__ << " Check failed: " << #x " " #OP " " #y << endl; \ - cerr << __FILE__ << ":" << __LINE__ << " Expected: " << _x << endl; \ - cerr << __FILE__ << ":" << __LINE__ << " Actual: " << _y << endl; \ - abort(); \ - } } while (false) - -#define CHECK(x) \ - do {if (!(x)) { \ - cerr << __FILE__ << ":" << __LINE__ << " Check failed: " << #x << endl; \ - abort(); \ - } } while (false) - -#include <string> -using std::string; - -#include <vector> -using std::vector; - -inline string CommandToString(const vector<const char*> &v) { - string s(v[0]); - for (size_t i = 1; i < v.size() && v[i] != NULL; i++) { - s.append(" "); - s.append(v[i]); - } - return s; -} - -#include <iostream> -using std::cerr; -using std::endl; -using std::ostream; - -#include <map> -using std::map; -using std::pair; - -#include <ext/hash_map> -using __gnu_cxx::hash_map; - -#include <list> -using std::list; - -template <typename T, typename U> -pair<T, U> make_pair(const T& t, const U& u) { - return pair<T, U>(t, u); -} - -class Constants { -public: - // TODO: need to repeat the tests with different block sizes - // 1 << 7 triggers some bugs, 1 << 20 triggers others. - // - //static const xoff_t BLOCK_SIZE = 1 << 20; - static const xoff_t BLOCK_SIZE = 1 << 7; -}; - -using std::min; - -#include "random.h" -using regtest::MTRandom; -using regtest::MTRandom8; - -#include "segment.h" -using regtest::Segment; - -#include "modify.h" -using regtest::Mutator; -using regtest::ChangeList; -using regtest::Change; -using regtest::ChangeListMutator; -using regtest::Modify1stByte; - -#include "file.h" -using regtest::Block; -using regtest::BlockIterator; -using regtest::ExtFile; -using regtest::FileSpec; -using regtest::TmpFile; - -#include "cmp.h" -using regtest::CmpDifferentBytes; - -#include "sizes.h" -using regtest::SizeIterator; -using regtest::SmallSizes; -using regtest::LargeSizes; - -#include "delta.h" -using regtest::Delta; diff --git a/xdelta3-cfgs.h b/xdelta3-cfgs.h deleted file mode 100644 index b13f7b0..0000000 --- a/xdelta3-cfgs.h +++ /dev/null @@ -1,173 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/****************************************************************** - SOFT string matcher - ******************************************************************/ - -#if XD3_BUILD_SOFT - -#define TEMPLATE soft -#define LLOOK stream->smatcher.large_look -#define LSTEP stream->smatcher.large_step -#define SLOOK stream->smatcher.small_look -#define SCHAIN stream->smatcher.small_chain -#define SLCHAIN stream->smatcher.small_lchain -#define MAXLAZY stream->smatcher.max_lazy -#define LONGENOUGH stream->smatcher.long_enough - -#define SOFTCFG 1 -#include "xdelta3.c" -#undef SOFTCFG - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -#define SOFTCFG 0 - -/************************************************************ - FASTEST string matcher - **********************************************************/ -#if XD3_BUILD_FASTEST -#define TEMPLATE fastest -#define LLOOK 9 -#define LSTEP 26 -#define SLOOK 4U -#define SCHAIN 1 -#define SLCHAIN 1 -#define MAXLAZY 6 -#define LONGENOUGH 6 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/************************************************************ - FASTER string matcher - **********************************************************/ -#if XD3_BUILD_FASTER -#define TEMPLATE faster -#define LLOOK 9 -#define LSTEP 15 -#define SLOOK 4U -#define SCHAIN 1 -#define SLCHAIN 1 -#define MAXLAZY 18 -#define LONGENOUGH 18 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/****************************************************** - FAST string matcher - ********************************************************/ -#if XD3_BUILD_FAST -#define TEMPLATE fast -#define LLOOK 9 -#define LSTEP 8 -#define SLOOK 4U -#define SCHAIN 4 -#define SLCHAIN 1 -#define MAXLAZY 18 -#define LONGENOUGH 35 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/************************************************** - SLOW string matcher - **************************************************************/ -#if XD3_BUILD_SLOW -#define TEMPLATE slow -#define LLOOK 9 -#define LSTEP 2 -#define SLOOK 4U -#define SCHAIN 44 -#define SLCHAIN 13 -#define MAXLAZY 90 -#define LONGENOUGH 70 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/******************************************************** - DEFAULT string matcher - ************************************************************/ -#if XD3_BUILD_DEFAULT -#define TEMPLATE default -#define LLOOK 9 -#define LSTEP 3 -#define SLOOK 4U -#define SCHAIN 8 -#define SLCHAIN 2 -#define MAXLAZY 36 -#define LONGENOUGH 70 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif diff --git a/xdelta3-decode.h b/xdelta3-decode.h deleted file mode 100644 index bf2b0b1..0000000 --- a/xdelta3-decode.h +++ /dev/null @@ -1,1115 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _XDELTA3_DECODE_H_ -#define _XDELTA3_DECODE_H_ - -#define SRCORTGT(x) ((((x) & VCD_SRCORTGT) == VCD_SOURCE) ? \ - VCD_SOURCE : ((((x) & VCD_SRCORTGT) == \ - VCD_TARGET) ? VCD_TARGET : 0)) - -/* Initialize the decoder for a new window. The dec_tgtlen value is - * preserved across successive window decodings, and the update to - * dec_winstart is delayed until a new window actually starts. This - * is to avoid throwing an error due to overflow until the last - * possible moment. This makes it possible to encode exactly 4GB - * through a 32-bit encoder. */ -static int -xd3_decode_init_window (xd3_stream *stream) -{ - stream->dec_cpylen = 0; - stream->dec_cpyoff = 0; - stream->dec_cksumbytes = 0; - - xd3_init_cache (& stream->acache); - - return 0; -} - -/* Allocates buffer space for the target window and possibly the - * VCD_TARGET copy-window. Also sets the base of the two copy - * segments. */ -static int -xd3_decode_setup_buffers (xd3_stream *stream) -{ - /* If VCD_TARGET is set then the previous buffer may be reused. */ - if (stream->dec_win_ind & VCD_TARGET) - { - /* But this implementation only supports copying from the last - * target window. If the offset is outside that range, it can't - * be done. */ - if (stream->dec_cpyoff < stream->dec_laststart) - { - stream->msg = "unsupported VCD_TARGET offset"; - return XD3_INVALID_INPUT; - } - - /* See if the two windows are the same. This indicates the - * first time VCD_TARGET is used. This causes a second buffer - * to be allocated, after that the two are swapped in the - * DEC_FINISH case. */ - if (stream->dec_lastwin == stream->next_out) - { - stream->next_out = NULL; - stream->space_out = 0; - } - - // TODO: VCD_TARGET mode, this is broken - stream->dec_cpyaddrbase = stream->dec_lastwin + - (usize_t) (stream->dec_cpyoff - stream->dec_laststart); - } - - /* See if the current output window is large enough. */ - if (stream->space_out < stream->dec_tgtlen) - { - xd3_free (stream, stream->dec_buffer); - - stream->space_out = - xd3_round_blksize (stream->dec_tgtlen, XD3_ALLOCSIZE); - - if ((stream->dec_buffer = - (uint8_t*) xd3_alloc (stream, stream->space_out, 1)) == NULL) - { - return ENOMEM; - } - - stream->next_out = stream->dec_buffer; - } - - /* dec_tgtaddrbase refers to an invalid base address, but it is - * always used with a sufficiently large instruction offset (i.e., - * beyond the copy window). This condition is enforced by - * xd3_decode_output_halfinst. */ - stream->dec_tgtaddrbase = stream->next_out - stream->dec_cpylen; - - return 0; -} - -static int -xd3_decode_allocate (xd3_stream *stream, - usize_t size, - uint8_t **buf_ptr, - usize_t *buf_alloc) -{ - if (*buf_ptr != NULL && *buf_alloc < size) - { - xd3_free (stream, *buf_ptr); - *buf_ptr = NULL; - } - - if (*buf_ptr == NULL) - { - *buf_alloc = xd3_round_blksize (size, XD3_ALLOCSIZE); - - if ((*buf_ptr = (uint8_t*) xd3_alloc (stream, *buf_alloc, 1)) == NULL) - { - return ENOMEM; - } - } - - return 0; -} - -static int -xd3_decode_section (xd3_stream *stream, - xd3_desect *section, - xd3_decode_state nstate, - int copy) -{ - XD3_ASSERT (section->pos <= section->size); - XD3_ASSERT (stream->dec_state != nstate); - - if (section->pos < section->size) - { - usize_t sect_take; - - if (stream->avail_in == 0) - { - return XD3_INPUT; - } - - if ((copy == 0) && (section->pos == 0)) - { - /* No allocation/copy needed */ - section->buf = stream->next_in; - sect_take = section->size; - } - else - { - usize_t sect_need = section->size - section->pos; - - /* Allocate and copy */ - sect_take = min (sect_need, stream->avail_in); - - if (section->pos == 0) - { - int ret; - - if ((ret = xd3_decode_allocate (stream, - section->size, - & section->copied1, - & section->alloc1))) - { - return ret; - } - - section->buf = section->copied1; - } - - memcpy (section->copied1 + section->pos, - stream->next_in, - sect_take); - } - - section->pos += sect_take; - - stream->dec_winbytes += sect_take; - - DECODE_INPUT (sect_take); - } - - if (section->pos < section->size) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - XD3_ASSERT (section->pos == section->size); - - stream->dec_state = nstate; - section->buf_max = section->buf + section->size; - section->pos = 0; - return 0; -} - -/* Decode the size and address for half of an instruction (i.e., a - * single opcode). This updates the stream->dec_position, which are - * bytes already output prior to processing this instruction. Perform - * bounds checking for sizes and copy addresses, which uses the - * dec_position (which is why these checks are done here). */ -static int -xd3_decode_parse_halfinst (xd3_stream *stream, xd3_hinst *inst) -{ - int ret; - - /* If the size from the instruction table is zero then read a size value. */ - if ((inst->size == 0) && - (ret = xd3_read_size (stream, - & stream->inst_sect.buf, - stream->inst_sect.buf_max, - & inst->size))) - { - return XD3_INVALID_INPUT; - } - - /* For copy instructions, read address. */ - if (inst->type >= XD3_CPY) - { - IF_DEBUG2 ({ - static int cnt = 0; - DP(RINT "DECODE:%u: COPY at %"Q"u (winoffset %u) size %u winaddr %u\n", - cnt++, - stream->total_out + (stream->dec_position - - stream->dec_cpylen), - (stream->dec_position - stream->dec_cpylen), - inst->size, - inst->addr); - }); - - if ((ret = xd3_decode_address (stream, - stream->dec_position, - inst->type - XD3_CPY, - & stream->addr_sect.buf, - stream->addr_sect.buf_max, - & inst->addr))) - { - return ret; - } - - /* Cannot copy an address before it is filled-in. */ - if (inst->addr >= stream->dec_position) - { - stream->msg = "address too large"; - return XD3_INVALID_INPUT; - } - - /* Check: a VCD_TARGET or VCD_SOURCE copy cannot exceed the remaining - * buffer space in its own segment. */ - if (inst->addr < stream->dec_cpylen && - inst->addr + inst->size > stream->dec_cpylen) - { - stream->msg = "size too large"; - return XD3_INVALID_INPUT; - } - } - else - { - IF_DEBUG2 ({ - if (inst->type == XD3_ADD) - { - static int cnt; - DP(RINT "DECODE:%d: ADD at %"Q"u (winoffset %u) size %u\n", - cnt++, - (stream->total_out + stream->dec_position - stream->dec_cpylen), - stream->dec_position - stream->dec_cpylen, - inst->size); - } - else - { - static int cnt; - XD3_ASSERT (inst->type == XD3_RUN); - DP(RINT "DECODE:%d: RUN at %"Q"u (winoffset %u) size %u\n", - cnt++, - stream->total_out + stream->dec_position - stream->dec_cpylen, - stream->dec_position - stream->dec_cpylen, - inst->size); - } - }); - } - - /* Check: The instruction will not overflow the output buffer. */ - if (stream->dec_position + inst->size > stream->dec_maxpos) - { - stream->msg = "size too large"; - return XD3_INVALID_INPUT; - } - - stream->dec_position += inst->size; - return 0; -} - -/* Decode a single opcode and then decode the two half-instructions. */ -static int -xd3_decode_instruction (xd3_stream *stream) -{ - int ret; - const xd3_dinst *inst; - - if (stream->inst_sect.buf == stream->inst_sect.buf_max) - { - stream->msg = "instruction underflow"; - return XD3_INVALID_INPUT; - } - - inst = &stream->code_table[*stream->inst_sect.buf++]; - - stream->dec_current1.type = inst->type1; - stream->dec_current2.type = inst->type2; - stream->dec_current1.size = inst->size1; - stream->dec_current2.size = inst->size2; - - /* For each instruction with a real operation, decode the - * corresponding size and addresses if necessary. Assume a - * code-table may have NOOP in either position, although this is - * unlikely. */ - if (inst->type1 != XD3_NOOP && - (ret = xd3_decode_parse_halfinst (stream, & stream->dec_current1))) - { - return ret; - } - if (inst->type2 != XD3_NOOP && - (ret = xd3_decode_parse_halfinst (stream, & stream->dec_current2))) - { - return ret; - } - return 0; -} - -/* Output the result of a single half-instruction. OPT: This the - decoder hotspot. */ -static int -xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst) -{ - /* To make this reentrant, set take = min (inst->size, available - space)... */ - usize_t take = inst->size; - - XD3_ASSERT (inst->type != XD3_NOOP); - - switch (inst->type) - { - case XD3_RUN: - { - /* Only require a single data byte. */ - if (stream->data_sect.buf == stream->data_sect.buf_max) - { - stream->msg = "data underflow"; - return XD3_INVALID_INPUT; - } - - memset (stream->next_out + stream->avail_out, - stream->data_sect.buf[0], - take); - - stream->data_sect.buf += 1; - stream->avail_out += take; - inst->type = XD3_NOOP; - break; - } - case XD3_ADD: - { - /* Require at least TAKE data bytes. */ - if (stream->data_sect.buf + take > stream->data_sect.buf_max) - { - stream->msg = "data underflow"; - return XD3_INVALID_INPUT; - } - - memcpy (stream->next_out + stream->avail_out, - stream->data_sect.buf, - take); - - stream->data_sect.buf += take; - stream->avail_out += take; - inst->type = XD3_NOOP; - break; - } - default: - { - usize_t i; - const uint8_t *src; - uint8_t *dst; - - /* See if it copies from the VCD_TARGET/VCD_SOURCE window or - * the target window. Out-of-bounds checks for the addresses - * and sizes are performed in xd3_decode_parse_halfinst. */ - if (inst->addr < stream->dec_cpylen) - { - if (stream->dec_win_ind & VCD_TARGET) - { - /* For VCD_TARGET we know the entire range is - * in-memory, as established by - * decode_setup_buffers. - * - * TODO: this is totally bogus, VCD_TARGET won't work. - */ - src = stream->dec_cpyaddrbase + inst->addr; - inst->type = XD3_NOOP; - inst->size = 0; - } - else - { - /* In this case we have to read a source block, which - * could return control to the caller. We need to - * know the first block number needed for this - * copy. */ - xd3_source *source; - xoff_t block; - usize_t blkoff; - usize_t blksize; - int ret; - - more: - - source = stream->src; - block = source->cpyoff_blocks; - blkoff = source->cpyoff_blkoff + inst->addr; - blksize = source->blksize; - - while (blkoff >= blksize) - { - block += 1; - blkoff -= blksize; - } - - if ((ret = xd3_getblk (stream, block))) - { - /* could be a XD3_GETSRCBLK failure. */ - if (ret == XD3_TOOFARBACK) - { - ret = XD3_INTERNAL; - } - return ret; - } - - src = source->curblk + blkoff; - - /* This block either contains enough data or the source file - * is short. */ - if ((source->onblk != blksize) && - (blkoff + take > source->onblk)) - { - stream->msg = "source file too short"; - return XD3_INVALID_INPUT; - - } - - XD3_ASSERT (blkoff != blksize); - - if (blkoff + take <= blksize) - { - inst->type = XD3_NOOP; - inst->size = 0; - } - else - { - /* This block doesn't contain all the data, modify - * the instruction, do not set to XD3_NOOP. */ - take = blksize - blkoff; - inst->size -= take; - inst->addr += take; - } - } - } - else - { - /* For a target-window copy, we know the entire range is - * in-memory. The dec_tgtaddrbase is negatively offset by - * dec_cpylen because the addresses start beyond that - * point. */ - src = stream->dec_tgtaddrbase + inst->addr; - inst->type = XD3_NOOP; - inst->size = 0; - } - - dst = stream->next_out + stream->avail_out; - - stream->avail_out += take; - - /* Can't just memcpy here due to possible overlap. */ - for (i = take; i != 0; i -= 1) - { - *dst++ = *src++; - } - - take = inst->size; - - /* If there is more to copy, call getblk again. */ - if (inst->type != XD3_NOOP) - { - XD3_ASSERT (take > 0); - goto more; - } - else - { - XD3_ASSERT (take == 0); - } - } - } - - return 0; -} - -static int -xd3_decode_finish_window (xd3_stream *stream) -{ - stream->dec_winbytes = 0; - stream->dec_state = DEC_FINISH; - - stream->data_sect.pos = 0; - stream->inst_sect.pos = 0; - stream->addr_sect.pos = 0; - - return XD3_OUTPUT; -} - -static int -xd3_decode_secondary_sections (xd3_stream *secondary_stream) -{ -#if SECONDARY_ANY - int ret; -#define DECODE_SECONDARY_SECTION(UPPER,LOWER) \ - ((secondary_stream->dec_del_ind & VCD_ ## UPPER ## COMP) && \ - (ret = xd3_decode_secondary (secondary_stream, \ - & secondary_stream-> LOWER ## _sect, \ - & xd3_sec_ ## LOWER (secondary_stream)))) - - if (DECODE_SECONDARY_SECTION (DATA, data) || - DECODE_SECONDARY_SECTION (INST, inst) || - DECODE_SECONDARY_SECTION (ADDR, addr)) - { - return ret; - } -#undef DECODE_SECONDARY_SECTION -#endif - return 0; -} - -static int -xd3_decode_sections (xd3_stream *stream) -{ - usize_t need, more, take; - int copy, ret; - - if ((stream->flags & XD3_JUST_HDR) != 0) - { - /* Nothing left to do. */ - return xd3_decode_finish_window (stream); - } - - /* To avoid copying, need this much data available */ - need = (stream->inst_sect.size + - stream->addr_sect.size + - stream->data_sect.size); - - /* The window may be entirely processed. */ - XD3_ASSERT (stream->dec_winbytes <= need); - - /* Compute how much more input is needed. */ - more = (need - stream->dec_winbytes); - - /* How much to consume. */ - take = min (more, stream->avail_in); - - /* See if the input is completely available, to avoid copy. */ - copy = (take != more); - - /* If the window is skipped... */ - if ((stream->flags & XD3_SKIP_WINDOW) != 0) - { - /* Skip the available input. */ - DECODE_INPUT (take); - - stream->dec_winbytes += take; - - if (copy) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - return xd3_decode_finish_window (stream); - } - - /* Process all but the DATA section. */ - switch (stream->dec_state) - { - default: - stream->msg = "internal error"; - return XD3_INVALID_INPUT; - - case DEC_DATA: - if ((ret = xd3_decode_section (stream, & stream->data_sect, - DEC_INST, copy))) { return ret; } - case DEC_INST: - if ((ret = xd3_decode_section (stream, & stream->inst_sect, - DEC_ADDR, copy))) { return ret; } - case DEC_ADDR: - if ((ret = xd3_decode_section (stream, & stream->addr_sect, - DEC_EMIT, copy))) { return ret; } - } - - XD3_ASSERT (stream->dec_winbytes == need); - - if ((ret = xd3_decode_secondary_sections (stream))) { return ret; } - - if (stream->flags & XD3_SKIP_EMIT) - { - return xd3_decode_finish_window (stream); - } - - /* OPT: A possible optimization is to avoid allocating memory in - * decode_setup_buffers and to avoid a large memcpy when the window - * consists of a single VCD_SOURCE copy instruction. The only - * potential problem is if the following window is a VCD_TARGET, - * then you need to remember... */ - if ((ret = xd3_decode_setup_buffers (stream))) { return ret; } - - return 0; -} - -static int -xd3_decode_emit (xd3_stream *stream) -{ - int ret; - - /* Produce output: originally structured to allow reentrant code - * that fills as much of the output buffer as possible, but VCDIFF - * semantics allows to copy from anywhere from the target window, so - * instead allocate a sufficiently sized buffer after the target - * window length is decoded. - * - * This code still needs to be reentrant to allow XD3_GETSRCBLK to - * return control. This is handled by setting the - * stream->dec_currentN instruction types to XD3_NOOP after they - * have been processed. */ - XD3_ASSERT (! (stream->flags & XD3_SKIP_EMIT)); - XD3_ASSERT (stream->dec_tgtlen <= stream->space_out); - - while (stream->inst_sect.buf != stream->inst_sect.buf_max || - stream->dec_current1.type != XD3_NOOP || - stream->dec_current2.type != XD3_NOOP) - { - /* Decode next instruction pair. */ - if ((stream->dec_current1.type == XD3_NOOP) && - (stream->dec_current2.type == XD3_NOOP) && - (ret = xd3_decode_instruction (stream))) { return ret; } - - /* Output for each instruction. */ - if ((stream->dec_current1.type != XD3_NOOP) && - (ret = xd3_decode_output_halfinst (stream, & stream->dec_current1))) - { - return ret; - } - - if ((stream->dec_current2.type != XD3_NOOP) && - (ret = xd3_decode_output_halfinst (stream, & stream->dec_current2))) - { - return ret; - } - } - - if (stream->avail_out != stream->dec_tgtlen) - { - IF_DEBUG1 (DP(RINT "AVAIL_OUT(%d) != DEC_TGTLEN(%d)\n", - stream->avail_out, stream->dec_tgtlen)); - stream->msg = "wrong window length"; - return XD3_INVALID_INPUT; - } - - if (stream->data_sect.buf != stream->data_sect.buf_max) - { - stream->msg = "extra data section"; - return XD3_INVALID_INPUT; - } - - if (stream->addr_sect.buf != stream->addr_sect.buf_max) - { - stream->msg = "extra address section"; - return XD3_INVALID_INPUT; - } - - /* OPT: Should cksum computation be combined with the above loop? */ - if ((stream->dec_win_ind & VCD_ADLER32) != 0 && - (stream->flags & XD3_ADLER32_NOVER) == 0) - { - uint32_t a32 = adler32 (1L, stream->next_out, stream->avail_out); - - if (a32 != stream->dec_adler32) - { - stream->msg = "target window checksum mismatch"; - return XD3_INVALID_INPUT; - } - } - - /* Finished with a window. */ - return xd3_decode_finish_window (stream); -} - -int -xd3_decode_input (xd3_stream *stream) -{ - int ret; - - if (stream->enc_state != 0) - { - stream->msg = "encoder/decoder transition"; - return XD3_INVALID_INPUT; - } - -#define BYTE_CASE(expr,x,nstate) \ - do { \ - if ( (expr) && \ - ((ret = xd3_decode_byte (stream, & (x))) != 0) ) { return ret; } \ - stream->dec_state = (nstate); \ - } while (0) - -#define OFFSET_CASE(expr,x,nstate) \ - do { \ - if ( (expr) && \ - ((ret = xd3_decode_offset (stream, & (x))) != 0) ) { return ret; } \ - stream->dec_state = (nstate); \ - } while (0) - -#define SIZE_CASE(expr,x,nstate) \ - do { \ - if ( (expr) && \ - ((ret = xd3_decode_size (stream, & (x))) != 0) ) { return ret; } \ - stream->dec_state = (nstate); \ - } while (0) - - switch (stream->dec_state) - { - case DEC_VCHEAD: - { - if ((ret = xd3_decode_bytes (stream, stream->dec_magic, - & stream->dec_magicbytes, 4))) - { - return ret; - } - - if (stream->dec_magic[0] != VCDIFF_MAGIC1 || - stream->dec_magic[1] != VCDIFF_MAGIC2 || - stream->dec_magic[2] != VCDIFF_MAGIC3) - { - stream->msg = "not a VCDIFF input"; - return XD3_INVALID_INPUT; - } - - if (stream->dec_magic[3] != 0) - { - stream->msg = "VCDIFF input version > 0 is not supported"; - return XD3_INVALID_INPUT; - } - - stream->dec_state = DEC_HDRIND; - } - case DEC_HDRIND: - { - if ((ret = xd3_decode_byte (stream, & stream->dec_hdr_ind))) - { - return ret; - } - - if ((stream->dec_hdr_ind & VCD_INVHDR) != 0) - { - stream->msg = "unrecognized header indicator bits set"; - return XD3_INVALID_INPUT; - } - - stream->dec_state = DEC_SECONDID; - } - - case DEC_SECONDID: - /* Secondary compressor ID: only if VCD_SECONDARY is set */ - if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) - { - BYTE_CASE (1, stream->dec_secondid, DEC_TABLEN); - - switch (stream->dec_secondid) - { - case VCD_FGK_ID: - FGK_CASE (stream); - case VCD_DJW_ID: - DJW_CASE (stream); - default: - stream->msg = "unknown secondary compressor ID"; - return XD3_INVALID_INPUT; - } - } - - case DEC_TABLEN: - /* Length of code table data: only if VCD_CODETABLE is set */ - SIZE_CASE ((stream->dec_hdr_ind & VCD_CODETABLE) != 0, - stream->dec_codetblsz, DEC_NEAR); - - /* The codetblsz counts the two NEAR/SAME bytes */ - if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) { - if (stream->dec_codetblsz <= 2) { - stream->msg = "invalid code table size"; - return ENOMEM; - } - stream->dec_codetblsz -= 2; - } - case DEC_NEAR: - /* Near modes: only if VCD_CODETABLE is set */ - BYTE_CASE((stream->dec_hdr_ind & VCD_CODETABLE) != 0, - stream->acache.s_near, DEC_SAME); - case DEC_SAME: - /* Same modes: only if VCD_CODETABLE is set */ - BYTE_CASE((stream->dec_hdr_ind & VCD_CODETABLE) != 0, - stream->acache.s_same, DEC_TABDAT); - case DEC_TABDAT: - /* Compressed code table data */ - - if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) - { - /* Get the code table data. */ - if ((stream->dec_codetbl == NULL) && - (stream->dec_codetbl = - (uint8_t*) xd3_alloc (stream, - stream->dec_codetblsz, 1)) == NULL) - { - return ENOMEM; - } - - if ((ret = xd3_decode_bytes (stream, stream->dec_codetbl, - & stream->dec_codetblbytes, - stream->dec_codetblsz))) - { - return ret; - } - - if ((ret = xd3_apply_table_encoding (stream, stream->dec_codetbl, - stream->dec_codetblbytes))) - { - return ret; - } - } - else - { - /* Use the default table. */ - stream->acache.s_near = __rfc3284_code_table_desc.near_modes; - stream->acache.s_same = __rfc3284_code_table_desc.same_modes; - stream->code_table = xd3_rfc3284_code_table (); - } - - if ((ret = xd3_alloc_cache (stream))) { return ret; } - - stream->dec_state = DEC_APPLEN; - - case DEC_APPLEN: - /* Length of application data */ - SIZE_CASE((stream->dec_hdr_ind & VCD_APPHEADER) != 0, - stream->dec_appheadsz, DEC_APPDAT); - - case DEC_APPDAT: - /* Application data */ - if (stream->dec_hdr_ind & VCD_APPHEADER) - { - /* Note: we add an additional byte for padding, to allow - 0-termination. */ - if ((stream->dec_appheader == NULL) && - (stream->dec_appheader = - (uint8_t*) xd3_alloc (stream, - stream->dec_appheadsz+1, 1)) == NULL) - { - return ENOMEM; - } - - stream->dec_appheader[stream->dec_appheadsz] = 0; - - if ((ret = xd3_decode_bytes (stream, stream->dec_appheader, - & stream->dec_appheadbytes, - stream->dec_appheadsz))) - { - return ret; - } - } - - stream->dec_hdrsize = stream->total_in; - stream->dec_state = DEC_WININD; - - case DEC_WININD: - { - /* Start of a window: the window indicator */ - if ((ret = xd3_decode_byte (stream, & stream->dec_win_ind))) - { - return ret; - } - - stream->current_window = stream->dec_window_count; - - if (XOFF_T_OVERFLOW (stream->dec_winstart, stream->dec_tgtlen)) - { - stream->msg = "decoder file offset overflow"; - return XD3_INVALID_INPUT; - } - - stream->dec_winstart += stream->dec_tgtlen; - - if ((stream->dec_win_ind & VCD_INVWIN) != 0) - { - stream->msg = "unrecognized window indicator bits set"; - return XD3_INVALID_INPUT; - } - - if ((ret = xd3_decode_init_window (stream))) { return ret; } - - stream->dec_state = DEC_CPYLEN; - - IF_DEBUG1 (DP(RINT "--------- TARGET WINDOW %"Q"u -----------\n", - stream->current_window)); - } - - case DEC_CPYLEN: - /* Copy window length: only if VCD_SOURCE or VCD_TARGET is set */ - SIZE_CASE(SRCORTGT (stream->dec_win_ind), stream->dec_cpylen, - DEC_CPYOFF); - - /* Set the initial, logical decoder position (HERE address) in - * dec_position. This is set to just after the source/copy - * window, as we are just about to output the first byte of - * target window. */ - stream->dec_position = stream->dec_cpylen; - - case DEC_CPYOFF: - /* Copy window offset: only if VCD_SOURCE or VCD_TARGET is set */ - OFFSET_CASE(SRCORTGT (stream->dec_win_ind), stream->dec_cpyoff, - DEC_ENCLEN); - - /* Copy offset and copy length may not overflow. */ - if (XOFF_T_OVERFLOW (stream->dec_cpyoff, stream->dec_cpylen)) - { - stream->msg = "decoder copy window overflows a file offset"; - return XD3_INVALID_INPUT; - } - - /* Check copy window bounds: VCD_TARGET window may not exceed - current position. */ - if ((stream->dec_win_ind & VCD_TARGET) && - (stream->dec_cpyoff + (xoff_t) stream->dec_cpylen > - stream->dec_winstart)) - { - stream->msg = "VCD_TARGET window out of bounds"; - return XD3_INVALID_INPUT; - } - - case DEC_ENCLEN: - /* Length of the delta encoding */ - SIZE_CASE(1, stream->dec_enclen, DEC_TGTLEN); - case DEC_TGTLEN: - /* Length of target window */ - SIZE_CASE(1, stream->dec_tgtlen, DEC_DELIND); - - /* Set the maximum decoder position, beyond which we should not - * decode any data. This is the maximum value for dec_position. - * This may not exceed the size of a usize_t. */ - if (USIZE_T_OVERFLOW (stream->dec_cpylen, stream->dec_tgtlen)) - { - stream->msg = "decoder target window overflows a usize_t"; - return XD3_INVALID_INPUT; - } - - /* Check for malicious files. */ - if (stream->dec_tgtlen > XD3_HARDMAXWINSIZE) - { - stream->msg = "hard window size exceeded"; - return XD3_INVALID_INPUT; - } - - stream->dec_maxpos = stream->dec_cpylen + stream->dec_tgtlen; - - case DEC_DELIND: - /* Delta indicator */ - BYTE_CASE(1, stream->dec_del_ind, DEC_DATALEN); - - if ((stream->dec_del_ind & VCD_INVDEL) != 0) - { - stream->msg = "unrecognized delta indicator bits set"; - return XD3_INVALID_INPUT; - } - - /* Delta indicator is only used with secondary compression. */ - if ((stream->dec_del_ind != 0) && (stream->sec_type == NULL)) - { - stream->msg = "invalid delta indicator bits set"; - return XD3_INVALID_INPUT; - } - - /* Section lengths */ - case DEC_DATALEN: - SIZE_CASE(1, stream->data_sect.size, DEC_INSTLEN); - case DEC_INSTLEN: - SIZE_CASE(1, stream->inst_sect.size, DEC_ADDRLEN); - case DEC_ADDRLEN: - SIZE_CASE(1, stream->addr_sect.size, DEC_CKSUM); - - case DEC_CKSUM: - /* Window checksum. */ - if ((stream->dec_win_ind & VCD_ADLER32) != 0) - { - int i; - - if ((ret = xd3_decode_bytes (stream, stream->dec_cksum, - & stream->dec_cksumbytes, 4))) - { - return ret; - } - - for (i = 0; i < 4; i += 1) - { - stream->dec_adler32 = - (stream->dec_adler32 << 8) | stream->dec_cksum[i]; - } - } - - stream->dec_state = DEC_DATA; - - /* Check dec_enclen for redundency, otherwise it is not really used. */ - { - usize_t enclen_check = - (1 + (xd3_sizeof_size (stream->dec_tgtlen) + - xd3_sizeof_size (stream->data_sect.size) + - xd3_sizeof_size (stream->inst_sect.size) + - xd3_sizeof_size (stream->addr_sect.size)) + - stream->data_sect.size + - stream->inst_sect.size + - stream->addr_sect.size + - ((stream->dec_win_ind & VCD_ADLER32) ? 4 : 0)); - - if (stream->dec_enclen != enclen_check) - { - stream->msg = "incorrect encoding length (redundent)"; - return XD3_INVALID_INPUT; - } - } - - /* Returning here gives the application a chance to inspect the - * header, skip the window, etc. */ - if (stream->current_window == 0) { return XD3_GOTHEADER; } - else { return XD3_WINSTART; } - - case DEC_DATA: - case DEC_INST: - case DEC_ADDR: - /* Next read the three sections. */ - if ((ret = xd3_decode_sections (stream))) { return ret; } - - case DEC_EMIT: - - /* To speed VCD_SOURCE block-address calculations, the source - * cpyoff_blocks and cpyoff_blkoff are pre-computed. */ - if (stream->dec_win_ind & VCD_SOURCE) - { - xd3_source *src = stream->src; - - if (src == NULL) - { - stream->msg = "source input required"; - return XD3_INVALID_INPUT; - } - - xd3_blksize_div(stream->dec_cpyoff, src, - &src->cpyoff_blocks, - &src->cpyoff_blkoff); - } - - /* xd3_decode_emit returns XD3_OUTPUT on every success. */ - if ((ret = xd3_decode_emit (stream)) == XD3_OUTPUT) - { - stream->total_out += (xoff_t) stream->avail_out; - } - - return ret; - - case DEC_FINISH: - { - if (stream->dec_win_ind & VCD_TARGET) - { - if (stream->dec_lastwin == NULL) - { - stream->dec_lastwin = stream->next_out; - stream->dec_lastspace = stream->space_out; - } - else - { - xd3_swap_uint8p (& stream->dec_lastwin, - & stream->next_out); - xd3_swap_usize_t (& stream->dec_lastspace, - & stream->space_out); - } - } - - stream->dec_lastlen = stream->dec_tgtlen; - stream->dec_laststart = stream->dec_winstart; - stream->dec_window_count += 1; - - /* Note: the updates to dec_winstart & current_window are - * deferred until after the next DEC_WININD byte is read. */ - stream->dec_state = DEC_WININD; - return XD3_WINFINISH; - } - - default: - stream->msg = "invalid state"; - return XD3_INVALID_INPUT; - } -} - -#endif // _XDELTA3_DECODE_H_ diff --git a/xdelta3-djw.h b/xdelta3-djw.h deleted file mode 100644 index 24f5b81..0000000 --- a/xdelta3-djw.h +++ /dev/null @@ -1,1828 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2002, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* TODO: This code needs a thorough round of commenting. There is - * some slop in the declaration of arrays, which are maybe one element - * larger than they need to be and comments would help clear it up. */ - -#ifndef _XDELTA3_DJW_H_ -#define _XDELTA3_DJW_H_ - -/* The following people deserve much credit for the algorithms and - * techniques contained in this file: - - Julian Seward - Bzip2 sources, implementation of the multi-table Huffman technique. - - Jean-loup Gailly and Mark Adler and L. Peter Deutsch - Zlib source code, RFC 1951 - - Daniel S. Hirschberg and Debra A. LeLewer - "Efficient Decoding of Prefix Codes" - Communications of the ACM, April 1990 33(4). - - David J. Wheeler - Program bred3.c, bexp3 and accompanying documents bred3.ps, huff.ps. - This contains the idea behind the multi-table Huffman and 1-2 coding - techniques. - ftp://ftp.cl.cam.ac.uk/users/djw3/ - -*/ - -/* OPT: during the multi-table iteration, pick the worst-overall - * performing table and replace it with exactly the frequencies of the - * worst-overall performing sector or N-worst performing sectors. */ - -/* REF: See xdfs-0.222 and xdfs-0.226 for some old experiments with - * the Bzip prefix coding strategy. xdfs-0.256 contains the last of - * the other-format tests, including RFC1950 and the RFC1950+MTF - * tests. */ - -#define DJW_MAX_CODELEN 20 /* Maximum length of an alphabet code. */ - -/* Code lengths are themselves code-length encoded, so the total number of - * codes is: [RUN_0, RUN_1, 1-DJW_MAX_CODELEN] */ -#define DJW_TOTAL_CODES (DJW_MAX_CODELEN+2) - -#define RUN_0 0 /* Symbols used in MTF+1/2 coding. */ -#define RUN_1 1 - -/* Number of code lengths always encoded (djw_encode_basic array) */ -#define DJW_BASIC_CODES 5 -#define DJW_RUN_CODES 2 /* Number of run codes */ - -/* Offset of extra codes */ -#define DJW_EXTRA_12OFFSET (DJW_BASIC_CODES + DJW_RUN_CODES) - -/* Number of optionally encoded code lengths (djw_encode_extra array) */ -#define DJW_EXTRA_CODES 15 - -/* Number of bits to code [0-DJW_EXTRA_CODES] */ -#define DJW_EXTRA_CODE_BITS 4 - -#define DJW_MAX_GROUPS 8 /* Max number of group coding tables */ -#define DJW_GROUP_BITS 3 /* Number of bits to code [1-DJW_MAX_GROUPS] */ - -#define DJW_SECTORSZ_MULT 5 /* Multiplier for encoded sectorsz */ -#define DJW_SECTORSZ_BITS 5 /* Number of bits to code group size */ -#define DJW_SECTORSZ_MAX ((1 << DJW_SECTORSZ_BITS) * DJW_SECTORSZ_MULT) - -/* Maximum number of iterations to find group tables. */ -#define DJW_MAX_ITER 6 -/* Minimum number of bits an iteration must reduce coding by. */ -#define DJW_MIN_IMPROVEMENT 20 - -/* Maximum code length of a prefix code length */ -#define DJW_MAX_CLCLEN 15 - -/* Number of bits to code [0-DJW_MAX_CLCLEN] */ -#define DJW_CLCLEN_BITS 4 - -#define DJW_MAX_GBCLEN 7 /* Maximum code length of a group selector */ - -/* Number of bits to code [0-DJW_MAX_GBCLEN] - * TODO: Actually, should never have zero code lengths here, or else a group - * went unused. Write a test for this: if a group goes unused, eliminate - * it? */ -#define DJW_GBCLEN_BITS 3 - -/* It has to save at least this many bits... */ -#define EFFICIENCY_BITS 16 - -typedef struct _djw_stream djw_stream; -typedef struct _djw_heapen djw_heapen; -typedef struct _djw_prefix djw_prefix; -typedef uint32_t djw_weight; - -struct _djw_heapen -{ - uint32_t depth; - uint32_t freq; - uint32_t parent; -}; - -struct _djw_prefix -{ - usize_t scount; - uint8_t *symbol; - usize_t mcount; - uint8_t *mtfsym; - uint8_t *repcnt; -}; - -struct _djw_stream -{ - int unused; -}; - -/* Each Huffman table consists of 256 "code length" (CLEN) codes, - * which are themselves Huffman coded after eliminating repeats and - * move-to-front coding. The prefix consists of all the CLEN codes in - * djw_encode_basic plus a 4-bit value stating how many of the - * djw_encode_extra codes are actually coded (the rest are presumed - * zero, or unused CLEN codes). - * - * These values of these two arrays were arrived at by studying the - * distribution of min and max clen over a collection of DATA, INST, - * and ADDR inputs. The goal is to specify the order of - * djw_extra_codes that is most likely to minimize the number of extra - * codes that must be encoded. - * - * Results: 158896 sections were counted by compressing files (window - * size 512K) listed with: `find / -type f ( -user jmacd -o -perm +444 - * )` - * - * The distribution of CLEN codes for each efficient invocation of the - * secondary compressor (taking the best number of groups/sector size) - * was recorded. Then we look at the distribution of min and max clen - * values, counting the number of times the value C_low is less than - * the min and C_high is greater than the max. Values >= C_high and - * <= C_low will not have their lengths coded. The results are sorted - * and the least likely 15 are placed into the djw_encode_extra[] - * array in order. These values are used as the initial MTF ordering. - - clow[1] = 155119 - clow[2] = 140325 - clow[3] = 84072 - --- - clow[4] = 7225 - clow[5] = 1093 - clow[6] = 215 - --- - chigh[4] = 1 - chigh[5] = 30 - chigh[6] = 218 - chigh[7] = 2060 - chigh[8] = 13271 - --- - chigh[9] = 39463 - chigh[10] = 77360 - chigh[11] = 118298 - chigh[12] = 141360 - chigh[13] = 154086 - chigh[14] = 157967 - chigh[15] = 158603 - chigh[16] = 158864 - chigh[17] = 158893 - chigh[18] = 158895 - chigh[19] = 158896 - chigh[20] = 158896 - -*/ - -static const uint8_t djw_encode_12extra[DJW_EXTRA_CODES] = - { - 9, 10, 3, 11, 2, 12, 13, 1, 14, 15, 16, 17, 18, 19, 20, - }; - -static const uint8_t djw_encode_12basic[DJW_BASIC_CODES] = - { - 4, 5, 6, 7, 8, - }; - -/*********************************************************************/ -/* DECLS */ -/*********************************************************************/ - -static djw_stream* djw_alloc (xd3_stream *stream); -static void djw_init (djw_stream *h); -static void djw_destroy (xd3_stream *stream, - djw_stream *h); - -#if XD3_ENCODER -static int xd3_encode_huff (xd3_stream *stream, - djw_stream *sec_stream, - xd3_output *input, - xd3_output *output, - xd3_sec_cfg *cfg); -#endif - -static int xd3_decode_huff (xd3_stream *stream, - djw_stream *sec_stream, - const uint8_t **input, - const uint8_t *const input_end, - uint8_t **output, - const uint8_t *const output_end); - -/*********************************************************************/ -/* HUFFMAN */ -/*********************************************************************/ - -static djw_stream* -djw_alloc (xd3_stream *stream) -{ - return xd3_alloc (stream, sizeof (djw_stream), 1); -} - -static void -djw_init (djw_stream *h) -{ - /* Fields are initialized prior to use. */ -} - -static void -djw_destroy (xd3_stream *stream, - djw_stream *h) -{ - xd3_free (stream, h); -} - - -/*********************************************************************/ -/* HEAP */ -/*********************************************************************/ - -static inline int -heap_less (const djw_heapen *a, const djw_heapen *b) -{ - return a->freq < b->freq || - (a->freq == b->freq && - a->depth < b->depth); -} - -static inline void -heap_insert (usize_t *heap, const djw_heapen *ents, usize_t p, const usize_t e) -{ - /* Insert ents[e] into next slot heap[p] */ - usize_t pp = p/2; /* P's parent */ - - while (heap_less (& ents[e], & ents[heap[pp]])) - { - heap[p] = heap[pp]; - p = pp; - pp = p/2; - } - - heap[p] = e; -} - -static inline djw_heapen* -heap_extract (usize_t *heap, const djw_heapen *ents, usize_t heap_last) -{ - usize_t smallest = heap[1]; - usize_t p, pc, t; - - /* Caller decrements heap_last, so heap_last+1 is the replacement elt. */ - heap[1] = heap[heap_last+1]; - - /* Re-heapify */ - for (p = 1; ; p = pc) - { - pc = p*2; - - /* Reached bottom of heap */ - if (pc > heap_last) { break; } - - /* See if second child is smaller. */ - if (pc < heap_last && heap_less (& ents[heap[pc+1]], & ents[heap[pc]])) - { - pc += 1; - } - - /* If pc is not smaller than p, heap property re-established. */ - if (! heap_less (& ents[heap[pc]], & ents[heap[p]])) { break; } - - t = heap[pc]; - heap[pc] = heap[p]; - heap[p] = t; - } - - return (djw_heapen*) & ents[smallest]; -} - -#if XD3_DEBUG -static void -heap_check (usize_t *heap, djw_heapen *ents, usize_t heap_last) -{ - usize_t i; - for (i = 1; i <= heap_last; i += 1) - { - /* Heap property: child not less than parent */ - XD3_ASSERT (! heap_less (& ents[heap[i]], & ents[heap[i/2]])); - - IF_DEBUG1 (DP(RINT "heap[%d] = %u\n", i, ents[heap[i]].freq)); - } -} -#endif - -/*********************************************************************/ -/* MTF, 1/2 */ -/*********************************************************************/ - -static inline usize_t -djw_update_mtf (uint8_t *mtf, usize_t mtf_i) -{ - int k; - usize_t sym = mtf[mtf_i]; - - for (k = mtf_i; k != 0; k -= 1) { mtf[k] = mtf[k-1]; } - - mtf[0] = sym; - return sym; -} - -static inline void -djw_update_1_2 (int *mtf_run, usize_t *mtf_i, - uint8_t *mtfsym, djw_weight *freq) -{ - int code; - - do - { - /* Offset by 1, since any number of RUN_ symbols implies run>0... */ - *mtf_run -= 1; - - code = (*mtf_run & 1) ? RUN_1 : RUN_0; - - mtfsym[(*mtf_i)++] = code; - freq[code] += 1; - *mtf_run >>= 1; - } - while (*mtf_run >= 1); - - *mtf_run = 0; -} - -static void -djw_init_clen_mtf_1_2 (uint8_t *clmtf) -{ - int i, cl_i = 0; - - clmtf[cl_i++] = 0; - for (i = 0; i < DJW_BASIC_CODES; i += 1) - { - clmtf[cl_i++] = djw_encode_12basic[i]; - } - for (i = 0; i < DJW_EXTRA_CODES; i += 1) - { - clmtf[cl_i++] = djw_encode_12extra[i]; - } -} - -/*********************************************************************/ -/* PREFIX CODES */ -/*********************************************************************/ -#if XD3_ENCODER -static usize_t -djw_build_prefix (const djw_weight *freq, uint8_t *clen, int asize, int maxlen) -{ - /* Heap with 0th entry unused, prefix tree with up to ALPHABET_SIZE-1 - * internal nodes, never more than ALPHABET_SIZE entries actually in the - * heap (minimum weight subtrees during prefix construction). First - * ALPHABET_SIZE entries are the actual symbols, next ALPHABET_SIZE-1 are - * internal nodes. */ - djw_heapen ents[ALPHABET_SIZE * 2]; - usize_t heap[ALPHABET_SIZE + 1]; - - usize_t heap_last; /* Index of the last _valid_ heap entry. */ - usize_t ents_size; /* Number of entries, including 0th fake entry */ - int overflow; /* Number of code lengths that overflow */ - uint32_t total_bits; - int i; - - IF_DEBUG (uint32_t first_bits = 0); - - /* Insert real symbol frequences. */ - for (i = 0; i < asize; i += 1) - { - ents[i+1].freq = freq[i]; - IF_DEBUG1 (DP(RINT "ents[%d] = freq[%d] = %d\n", - i+1, i, freq[i])); - } - - again: - - /* The loop is re-entered each time an overflow occurs. Re-initialize... */ - heap_last = 0; - ents_size = 1; - overflow = 0; - total_bits = 0; - - /* 0th entry terminates the while loop in heap_insert (it's the parent of - * the smallest element, always less-than) */ - heap[0] = 0; - ents[0].depth = 0; - ents[0].freq = 0; - - /* Initial heap. */ - for (i = 0; i < asize; i += 1, ents_size += 1) - { - ents[ents_size].depth = 0; - ents[ents_size].parent = 0; - - if (ents[ents_size].freq != 0) - { - heap_insert (heap, ents, ++heap_last, ents_size); - } - } - - IF_DEBUG (heap_check (heap, ents, heap_last)); - - /* Must be at least one symbol, or else we can't get here. */ - XD3_ASSERT (heap_last != 0); - - /* If there is only one symbol, fake a second to prevent zero-length - * codes. */ - if (heap_last == 1) - { - /* Pick either the first or last symbol. */ - int s = freq[0] ? asize-1 : 0; - ents[s+1].freq = 1; - goto again; - } - - /* Build prefix tree. */ - while (heap_last > 1) - { - djw_heapen *h1 = heap_extract (heap, ents, --heap_last); - djw_heapen *h2 = heap_extract (heap, ents, --heap_last); - - ents[ents_size].freq = h1->freq + h2->freq; - ents[ents_size].depth = 1 + max (h1->depth, h2->depth); - ents[ents_size].parent = 0; - - h1->parent = h2->parent = ents_size; - - heap_insert (heap, ents, ++heap_last, ents_size++); - } - - IF_DEBUG (heap_check (heap, ents, heap_last)); - - /* Now compute prefix code lengths, counting parents. */ - for (i = 1; i < asize+1; i += 1) - { - int b = 0; - - if (ents[i].freq != 0) - { - int p = i; - - while ((p = ents[p].parent) != 0) { b += 1; } - - if (b > maxlen) { overflow = 1; } - - total_bits += b * freq[i-1]; - } - - /* clen is 0-origin, unlike ents. */ - IF_DEBUG1 (DP(RINT "clen[%d] = %d\n", i-1, b)); - clen[i-1] = b; - } - - IF_DEBUG (if (first_bits == 0) first_bits = total_bits); - - if (! overflow) - { - IF_DEBUG1 (if (first_bits != total_bits) - { - DP(RINT "code length overflow changed %u bits\n", - (usize_t)(total_bits - first_bits)); - }); - return total_bits; - } - - /* OPT: There is a non-looping way to fix overflow shown in zlib, but this - * is easier (for now), as done in bzip2. */ - for (i = 1; i < asize+1; i += 1) - { - ents[i].freq = ents[i].freq / 2 + 1; - } - - goto again; -} - -static void -djw_build_codes (usize_t *codes, const uint8_t *clen, int asize, int abs_max) -{ - int i, l; - int min_clen = DJW_MAX_CODELEN; - int max_clen = 0; - usize_t code = 0; - - /* Find the min and max code length */ - for (i = 0; i < asize; i += 1) - { - if (clen[i] > 0 && clen[i] < min_clen) - { - min_clen = clen[i]; - } - - max_clen = max (max_clen, (int) clen[i]); - } - - XD3_ASSERT (max_clen <= abs_max); - - /* Generate a code for each symbol with the appropriate length. */ - for (l = min_clen; l <= max_clen; l += 1) - { - for (i = 0; i < asize; i += 1) - { - if (clen[i] == l) - { - codes[i] = code++; - } - } - - code <<= 1; - } - - IF_DEBUG1 ({ - for (i = 0; i < asize; i += 1) - { - DP(RINT "code[%d] = %u\n", i, codes[i]); - } - }); -} - -/*********************************************************************/ -/* MOVE-TO-FRONT */ -/*********************************************************************/ -static void -djw_compute_mtf_1_2 (djw_prefix *prefix, - uint8_t *mtf, - djw_weight *freq_out, - usize_t nsym) -{ - int i, j, k; - usize_t sym; - usize_t size = prefix->scount; - usize_t mtf_i = 0; - int mtf_run = 0; - - /* This +2 is for the RUN_0, RUN_1 codes */ - memset (freq_out, 0, sizeof (freq_out[0]) * (nsym+2)); - - for (i = 0; i < size; ) - { - /* OPT: Bzip optimizes this algorithm a little by effectively checking - * j==0 before the MTF update. */ - sym = prefix->symbol[i++]; - - for (j = 0; mtf[j] != sym; j += 1) { } - - XD3_ASSERT (j <= nsym); - - for (k = j; k >= 1; k -= 1) { mtf[k] = mtf[k-1]; } - - mtf[0] = sym; - - if (j == 0) - { - mtf_run += 1; - continue; - } - - if (mtf_run > 0) - { - djw_update_1_2 (& mtf_run, & mtf_i, prefix->mtfsym, freq_out); - } - - /* Non-zero symbols are offset by RUN_1 */ - prefix->mtfsym[mtf_i++] = j+RUN_1; - freq_out[j+RUN_1] += 1; - } - - if (mtf_run > 0) - { - djw_update_1_2 (& mtf_run, & mtf_i, prefix->mtfsym, freq_out); - } - - prefix->mcount = mtf_i; -} - -/* Counts character frequencies of the input buffer, returns the size. */ -static usize_t -djw_count_freqs (djw_weight *freq, xd3_output *input) -{ - xd3_output *in; - usize_t size = 0; - - memset (freq, 0, sizeof (freq[0]) * ALPHABET_SIZE); - - for (in = input; in; in = in->next_page) - { - const uint8_t *p = in->base; - const uint8_t *p_max = p + in->next; - - size += in->next; - - do - { - ++freq[*p]; - } - while (++p < p_max); - } - - IF_DEBUG1 ({int i; - DP(RINT "freqs: "); - for (i = 0; i < ALPHABET_SIZE; i += 1) - { - DP(RINT "%u ", freq[i]); - } - DP(RINT "\n");}); - - return size; -} - -static void -djw_compute_multi_prefix (int groups, - uint8_t clen[DJW_MAX_GROUPS][ALPHABET_SIZE], - djw_prefix *prefix) -{ - int gp, i; - - prefix->scount = ALPHABET_SIZE; - memcpy (prefix->symbol, clen[0], ALPHABET_SIZE); - - for (gp = 1; gp < groups; gp += 1) - { - for (i = 0; i < ALPHABET_SIZE; i += 1) - { - if (clen[gp][i] == 0) - { - continue; - } - - prefix->symbol[prefix->scount++] = clen[gp][i]; - } - } -} - -static void -djw_compute_prefix_1_2 (djw_prefix *prefix, djw_weight *freq) -{ - /* This +1 is for the 0 code-length. */ - uint8_t clmtf[DJW_MAX_CODELEN+1]; - - djw_init_clen_mtf_1_2 (clmtf); - - djw_compute_mtf_1_2 (prefix, clmtf, freq, DJW_MAX_CODELEN); -} - -static int -djw_encode_prefix (xd3_stream *stream, - xd3_output **output, - bit_state *bstate, - djw_prefix *prefix) -{ - int ret, i; - usize_t num_to_encode; - djw_weight clfreq[DJW_TOTAL_CODES]; - uint8_t clclen[DJW_TOTAL_CODES]; - usize_t clcode[DJW_TOTAL_CODES]; - - /* Move-to-front encode prefix symbols, count frequencies */ - djw_compute_prefix_1_2 (prefix, clfreq); - - /* Compute codes */ - djw_build_prefix (clfreq, clclen, DJW_TOTAL_CODES, DJW_MAX_CLCLEN); - djw_build_codes (clcode, clclen, DJW_TOTAL_CODES, DJW_MAX_CLCLEN); - - /* Compute number of extra codes beyond basic ones for this template. */ - num_to_encode = DJW_TOTAL_CODES; - while (num_to_encode > DJW_EXTRA_12OFFSET && clclen[num_to_encode-1] == 0) - { - num_to_encode -= 1; - } - XD3_ASSERT (num_to_encode - DJW_EXTRA_12OFFSET < (1 << DJW_EXTRA_CODE_BITS)); - - /* Encode: # of extra codes */ - if ((ret = xd3_encode_bits (stream, output, bstate, DJW_EXTRA_CODE_BITS, - num_to_encode - DJW_EXTRA_12OFFSET))) - { - return ret; - } - - /* Encode: MTF code lengths */ - for (i = 0; i < num_to_encode; i += 1) - { - if ((ret = xd3_encode_bits (stream, output, bstate, - DJW_CLCLEN_BITS, clclen[i]))) - { - return ret; - } - } - - /* Encode: CLEN code lengths */ - for (i = 0; i < prefix->mcount; i += 1) - { - usize_t mtf_sym = prefix->mtfsym[i]; - usize_t bits = clclen[mtf_sym]; - usize_t code = clcode[mtf_sym]; - - if ((ret = xd3_encode_bits (stream, output, bstate, bits, code))) - { - return ret; - } - } - - return 0; -} - -static void -djw_compute_selector_1_2 (djw_prefix *prefix, - usize_t groups, - djw_weight *gbest_freq) -{ - uint8_t grmtf[DJW_MAX_GROUPS]; - usize_t i; - - for (i = 0; i < groups; i += 1) { grmtf[i] = i; } - - djw_compute_mtf_1_2 (prefix, grmtf, gbest_freq, groups); -} - -static int -xd3_encode_howmany_groups (xd3_stream *stream, - xd3_sec_cfg *cfg, - usize_t input_size, - usize_t *ret_groups, - usize_t *ret_sector_size) -{ - usize_t cfg_groups = 0; - usize_t cfg_sector_size = 0; - usize_t sugg_groups = 0; - usize_t sugg_sector_size = 0; - - if (cfg->ngroups != 0) - { - if (cfg->ngroups < 0 || cfg->ngroups > DJW_MAX_GROUPS) - { - stream->msg = "invalid secondary encoder group number"; - return XD3_INTERNAL; - } - - cfg_groups = cfg->ngroups; - } - - if (cfg->sector_size != 0) - { - if (cfg->sector_size < DJW_SECTORSZ_MULT || - cfg->sector_size > DJW_SECTORSZ_MAX || - (cfg->sector_size % DJW_SECTORSZ_MULT) != 0) - { - stream->msg = "invalid secondary encoder sector size"; - return XD3_INTERNAL; - } - - cfg_sector_size = cfg->sector_size; - } - - if (cfg_groups == 0 || cfg_sector_size == 0) - { - /* These values were found empirically using xdelta3-tune around version - * xdfs-0.256. */ - switch (cfg->data_type) - { - case DATA_SECTION: - if (input_size < 1000) { sugg_groups = 1; sugg_sector_size = 0; } - else if (input_size < 4000) { sugg_groups = 2; sugg_sector_size = 10; } - else if (input_size < 7000) { sugg_groups = 3; sugg_sector_size = 10; } - else if (input_size < 10000) { sugg_groups = 4; sugg_sector_size = 10; } - else if (input_size < 25000) { sugg_groups = 5; sugg_sector_size = 10; } - else if (input_size < 50000) { sugg_groups = 7; sugg_sector_size = 20; } - else if (input_size < 100000) { sugg_groups = 8; sugg_sector_size = 30; } - else { sugg_groups = 8; sugg_sector_size = 70; } - break; - case INST_SECTION: - if (input_size < 7000) { sugg_groups = 1; sugg_sector_size = 0; } - else if (input_size < 10000) { sugg_groups = 2; sugg_sector_size = 50; } - else if (input_size < 25000) { sugg_groups = 3; sugg_sector_size = 50; } - else if (input_size < 50000) { sugg_groups = 6; sugg_sector_size = 40; } - else if (input_size < 100000) { sugg_groups = 8; sugg_sector_size = 40; } - else { sugg_groups = 8; sugg_sector_size = 40; } - break; - case ADDR_SECTION: - if (input_size < 9000) { sugg_groups = 1; sugg_sector_size = 0; } - else if (input_size < 25000) { sugg_groups = 2; sugg_sector_size = 130; } - else if (input_size < 50000) { sugg_groups = 3; sugg_sector_size = 130; } - else if (input_size < 100000) { sugg_groups = 5; sugg_sector_size = 130; } - else { sugg_groups = 7; sugg_sector_size = 130; } - break; - } - - if (cfg_groups == 0) - { - cfg_groups = sugg_groups; - } - - if (cfg_sector_size == 0) - { - cfg_sector_size = sugg_sector_size; - } - } - - if (cfg_groups != 1 && cfg_sector_size == 0) - { - switch (cfg->data_type) - { - case DATA_SECTION: - cfg_sector_size = 20; - break; - case INST_SECTION: - cfg_sector_size = 50; - break; - case ADDR_SECTION: - cfg_sector_size = 130; - break; - } - } - - (*ret_groups) = cfg_groups; - (*ret_sector_size) = cfg_sector_size; - - XD3_ASSERT (cfg_groups > 0 && cfg_groups <= DJW_MAX_GROUPS); - XD3_ASSERT (cfg_groups == 1 || - (cfg_sector_size >= DJW_SECTORSZ_MULT && - cfg_sector_size <= DJW_SECTORSZ_MAX)); - - return 0; -} - -static int -xd3_encode_huff (xd3_stream *stream, - djw_stream *h, - xd3_output *input, - xd3_output *output, - xd3_sec_cfg *cfg) -{ - int ret; - usize_t groups, sector_size; - bit_state bstate = BIT_STATE_ENCODE_INIT; - xd3_output *in; - int output_bits; - usize_t input_bits; - usize_t input_bytes; - usize_t initial_offset = output->next; - djw_weight real_freq[ALPHABET_SIZE]; - uint8_t *gbest = NULL; - uint8_t *gbest_mtf = NULL; - - input_bytes = djw_count_freqs (real_freq, input); - input_bits = input_bytes * 8; - - XD3_ASSERT (input_bytes > 0); - - if ((ret = xd3_encode_howmany_groups (stream, cfg, input_bytes, - & groups, & sector_size))) - { - return ret; - } - - if (0) - { - regroup: - /* Sometimes we dynamically decide there are too many groups. Arrive - * here. */ - output->next = initial_offset; - xd3_bit_state_encode_init (& bstate); - } - - /* Encode: # of groups (3 bits) */ - if ((ret = xd3_encode_bits (stream, & output, & bstate, - DJW_GROUP_BITS, groups-1))) { goto failure; } - - if (groups == 1) - { - /* Single Huffman group. */ - usize_t code[ALPHABET_SIZE]; /* Codes */ - uint8_t clen[ALPHABET_SIZE]; - uint8_t prefix_mtfsym[ALPHABET_SIZE]; - djw_prefix prefix; - - output_bits = - djw_build_prefix (real_freq, clen, ALPHABET_SIZE, DJW_MAX_CODELEN); - djw_build_codes (code, clen, ALPHABET_SIZE, DJW_MAX_CODELEN); - - if (output_bits + EFFICIENCY_BITS >= input_bits && ! cfg->inefficient) - { - goto nosecond; - } - - /* Encode: prefix */ - prefix.mtfsym = prefix_mtfsym; - prefix.symbol = clen; - prefix.scount = ALPHABET_SIZE; - - if ((ret = djw_encode_prefix (stream, & output, & bstate, & prefix))) - { - goto failure; - } - - if (output_bits + (8 * output->next) + EFFICIENCY_BITS >= - input_bits && ! cfg->inefficient) - { - goto nosecond; - } - - /* Encode: data */ - for (in = input; in; in = in->next_page) - { - const uint8_t *p = in->base; - const uint8_t *p_max = p + in->next; - - do - { - usize_t sym = *p++; - usize_t bits = clen[sym]; - - IF_DEBUG (output_bits -= bits); - - if ((ret = xd3_encode_bits (stream, & output, - & bstate, bits, code[sym]))) - { - goto failure; - } - } - while (p < p_max); - } - - XD3_ASSERT (output_bits == 0); - } - else - { - /* DJW Huffman */ - djw_weight evolve_freq[DJW_MAX_GROUPS][ALPHABET_SIZE]; - uint8_t evolve_clen[DJW_MAX_GROUPS][ALPHABET_SIZE]; - djw_weight left = input_bytes; - int gp; - int niter = 0; - usize_t select_bits; - usize_t sym1 = 0, sym2 = 0, s; - usize_t gcost[DJW_MAX_GROUPS]; - usize_t gbest_code[DJW_MAX_GROUPS+2]; - uint8_t gbest_clen[DJW_MAX_GROUPS+2]; - usize_t gbest_max = 1 + (input_bytes - 1) / sector_size; - int best_bits = 0; - usize_t gbest_no; - usize_t gpcnt; - const uint8_t *p; - IF_DEBUG1 (usize_t gcount[DJW_MAX_GROUPS]); - - /* Encode: sector size (5 bits) */ - if ((ret = xd3_encode_bits (stream, & output, & bstate, - DJW_SECTORSZ_BITS, - (sector_size/DJW_SECTORSZ_MULT)-1))) - { - goto failure; - } - - /* Dynamic allocation. */ - if (gbest == NULL) - { - if ((gbest = xd3_alloc (stream, gbest_max, 1)) == NULL) - { - ret = ENOMEM; - goto failure; - } - } - - if (gbest_mtf == NULL) - { - if ((gbest_mtf = xd3_alloc (stream, gbest_max, 1)) == NULL) - { - ret = ENOMEM; - goto failure; - } - } - - /* OPT: Some of the inner loops can be optimized, as shown in bzip2 */ - - /* Generate initial code length tables. */ - for (gp = 0; gp < groups; gp += 1) - { - djw_weight sum = 0; - djw_weight goal = left / (groups - gp); - - IF_DEBUG1 (usize_t nz = 0); - - /* Due to the single-code granularity of this distribution, it may - * be that we can't generate a distribution for each group. In that - * case subtract one group and try again. If (inefficient), we're - * testing group behavior, so don't mess things up. */ - if (goal == 0 && !cfg->inefficient) - { - IF_DEBUG1 (DP(RINT "too many groups (%u), dropping one\n", - groups)); - groups -= 1; - goto regroup; - } - - /* Sum == goal is possible when (cfg->inefficient)... */ - while (sum < goal) - { - XD3_ASSERT (sym2 < ALPHABET_SIZE); - IF_DEBUG1 (nz += real_freq[sym2] != 0); - sum += real_freq[sym2++]; - } - - IF_DEBUG1(DP(RINT "group %u has symbols %u..%u (%u non-zero) " - "(%u/%u = %.3f)\n", - gp, sym1, sym2, nz, sum, - input_bytes, sum / (double)input_bytes);); - - for (s = 0; s < ALPHABET_SIZE; s += 1) - { - evolve_clen[gp][s] = (s >= sym1 && s <= sym2) ? 1 : 16; - } - - left -= sum; - sym1 = sym2+1; - } - - repeat: - - niter += 1; - gbest_no = 0; - memset (evolve_freq, 0, sizeof (evolve_freq[0]) * groups); - IF_DEBUG1 (memset (gcount, 0, sizeof (gcount[0]) * groups)); - - /* For each input page (loop is irregular to allow non-pow2-size group - * size. */ - in = input; - p = in->base; - - /* For each group-size sector. */ - do - { - const uint8_t *p0 = p; - xd3_output *in0 = in; - usize_t best = 0; - usize_t winner = 0; - - /* Select best group for each sector, update evolve_freq. */ - memset (gcost, 0, sizeof (gcost[0]) * groups); - - /* For each byte in sector. */ - for (gpcnt = 0; gpcnt < sector_size; gpcnt += 1) - { - /* For each group. */ - for (gp = 0; gp < groups; gp += 1) - { - gcost[gp] += evolve_clen[gp][*p]; - } - - /* Check end-of-input-page. */ -# define GP_PAGE() \ - if (++p - in->base == in->next) \ - { \ - in = in->next_page; \ - if (in == NULL) { break; } \ - p = in->base; \ - } - - GP_PAGE (); - } - - /* Find min cost group for this sector */ - best = -1U; - for (gp = 0; gp < groups; gp += 1) - { - if (gcost[gp] < best) { best = gcost[gp]; winner = gp; } - } - - XD3_ASSERT(gbest_no < gbest_max); - gbest[gbest_no++] = winner; - IF_DEBUG1 (gcount[winner] += 1); - - p = p0; - in = in0; - - /* Update group frequencies. */ - for (gpcnt = 0; gpcnt < sector_size; gpcnt += 1) - { - evolve_freq[winner][*p] += 1; - - GP_PAGE (); - } - } - while (in != NULL); - - XD3_ASSERT (gbest_no == gbest_max); - - /* Recompute code lengths. */ - output_bits = 0; - for (gp = 0; gp < groups; gp += 1) - { - int i; - uint8_t evolve_zero[ALPHABET_SIZE]; - int any_zeros = 0; - - memset (evolve_zero, 0, sizeof (evolve_zero)); - - /* Cannot allow a zero clen when the real frequency is non-zero. - * Note: this means we are going to encode a fairly long code for - * these unused entries. An improvement would be to implement a - * NOTUSED code for when these are actually zero, but this requires - * another data structure (evolve_zero) since we don't know when - * evolve_freq[i] == 0... Briefly tested, looked worse. */ - for (i = 0; i < ALPHABET_SIZE; i += 1) - { - if (evolve_freq[gp][i] == 0 && real_freq[i] != 0) - { - evolve_freq[gp][i] = 1; - evolve_zero[i] = 1; - any_zeros = 1; - } - } - - output_bits += djw_build_prefix (evolve_freq[gp], evolve_clen[gp], - ALPHABET_SIZE, DJW_MAX_CODELEN); - - /* The above faking of frequencies does not matter for the last - * iteration, but we don't know when that is yet. However, it also - * breaks the output_bits computation. Necessary for accuracy, and - * for the (output_bits==0) assert after all bits are output. */ - if (any_zeros) - { - IF_DEBUG1 (usize_t save_total = output_bits); - - for (i = 0; i < ALPHABET_SIZE; i += 1) - { - if (evolve_zero[i]) { output_bits -= evolve_clen[gp][i]; } - } - - IF_DEBUG1 (DP(RINT "evolve_zero reduced %u bits in group %u\n", - save_total - output_bits, gp)); - } - } - - IF_DEBUG1( - DP(RINT "pass %u total bits: %u group uses: ", niter, output_bits); - for (gp = 0; gp < groups; gp += 1) { DP(RINT "%u ", gcount[gp]); } - DP(RINT "\n"); - ); - - /* End iteration. */ - - IF_DEBUG1 (if (niter > 1 && best_bits < output_bits) { - DP(RINT "iteration lost %u bits\n", output_bits - best_bits); }); - - if (niter == 1 || (niter < DJW_MAX_ITER && - (best_bits - output_bits) >= DJW_MIN_IMPROVEMENT)) - { - best_bits = output_bits; - goto repeat; - } - - /* Efficiency check. */ - if (output_bits + EFFICIENCY_BITS >= input_bits && ! cfg->inefficient) - { - goto nosecond; - } - - IF_DEBUG1 (DP(RINT "djw compression: %u -> %0.3f\n", - input_bytes, output_bits / 8.0)); - - /* Encode: prefix */ - { - uint8_t prefix_symbol[DJW_MAX_GROUPS * ALPHABET_SIZE]; - uint8_t prefix_mtfsym[DJW_MAX_GROUPS * ALPHABET_SIZE]; - uint8_t prefix_repcnt[DJW_MAX_GROUPS * ALPHABET_SIZE]; - djw_prefix prefix; - - prefix.symbol = prefix_symbol; - prefix.mtfsym = prefix_mtfsym; - prefix.repcnt = prefix_repcnt; - - djw_compute_multi_prefix (groups, evolve_clen, & prefix); - if ((ret = djw_encode_prefix (stream, & output, & bstate, & prefix))) - { - goto failure; - } - } - - /* Encode: selector frequencies */ - { - djw_weight gbest_freq[DJW_MAX_GROUPS+1]; - djw_prefix gbest_prefix; - usize_t i; - - gbest_prefix.scount = gbest_no; - gbest_prefix.symbol = gbest; - gbest_prefix.mtfsym = gbest_mtf; - - djw_compute_selector_1_2 (& gbest_prefix, groups, gbest_freq); - - select_bits = - djw_build_prefix (gbest_freq, gbest_clen, groups+1, DJW_MAX_GBCLEN); - djw_build_codes (gbest_code, gbest_clen, groups+1, DJW_MAX_GBCLEN); - - for (i = 0; i < groups+1; i += 1) - { - if ((ret = xd3_encode_bits (stream, & output, & bstate, - DJW_GBCLEN_BITS, gbest_clen[i]))) - { - goto failure; - } - } - - for (i = 0; i < gbest_prefix.mcount; i += 1) - { - usize_t gp_mtf = gbest_mtf[i]; - usize_t gp_sel_bits = gbest_clen[gp_mtf]; - usize_t gp_sel_code = gbest_code[gp_mtf]; - - XD3_ASSERT (gp_mtf < groups+1); - - if ((ret = xd3_encode_bits (stream, & output, & bstate, - gp_sel_bits, gp_sel_code))) - { - goto failure; - } - - IF_DEBUG (select_bits -= gp_sel_bits); - } - - XD3_ASSERT (select_bits == 0); - } - - /* Efficiency check. */ - if (output_bits + select_bits + (8 * output->next) + - EFFICIENCY_BITS >= input_bits && ! cfg->inefficient) - { - goto nosecond; - } - - /* Encode: data */ - { - usize_t evolve_code[DJW_MAX_GROUPS][ALPHABET_SIZE]; - usize_t sector = 0; - - /* Build code tables for each group. */ - for (gp = 0; gp < groups; gp += 1) - { - djw_build_codes (evolve_code[gp], evolve_clen[gp], - ALPHABET_SIZE, DJW_MAX_CODELEN); - } - - /* Now loop over the input. */ - in = input; - p = in->base; - - do - { - /* For each sector. */ - usize_t gp_best = gbest[sector]; - usize_t *gp_codes = evolve_code[gp_best]; - uint8_t *gp_clens = evolve_clen[gp_best]; - - XD3_ASSERT (sector < gbest_no); - - sector += 1; - - /* Encode the sector data. */ - for (gpcnt = 0; gpcnt < sector_size; gpcnt += 1) - { - usize_t sym = *p; - usize_t bits = gp_clens[sym]; - usize_t code = gp_codes[sym]; - - IF_DEBUG (output_bits -= bits); - - if ((ret = xd3_encode_bits (stream, & output, & bstate, - bits, code))) - { - goto failure; - } - - GP_PAGE (); - } - } - while (in != NULL); - - XD3_ASSERT (select_bits == 0); - XD3_ASSERT (output_bits == 0); - } - } - - ret = xd3_flush_bits (stream, & output, & bstate); - - if (0) - { - nosecond: - stream->msg = "secondary compression was inefficient"; - ret = XD3_NOSECOND; - } - - failure: - - xd3_free (stream, gbest); - xd3_free (stream, gbest_mtf); - return ret; -} -#endif /* XD3_ENCODER */ - -/*********************************************************************/ -/* DECODE */ -/*********************************************************************/ - -static void -djw_build_decoder (xd3_stream *stream, - usize_t asize, - usize_t abs_max, - const uint8_t *clen, - uint8_t *inorder, - usize_t *base, - usize_t *limit, - usize_t *min_clenp, - usize_t *max_clenp) -{ - int i, l; - const uint8_t *ci; - usize_t nr_clen [DJW_TOTAL_CODES]; - usize_t tmp_base[DJW_TOTAL_CODES]; - int min_clen; - int max_clen; - - /* Assumption: the two temporary arrays are large enough to hold abs_max. */ - XD3_ASSERT (abs_max <= DJW_MAX_CODELEN); - - /* This looks something like the start of zlib's inftrees.c */ - memset (nr_clen, 0, sizeof (nr_clen[0]) * (abs_max+1)); - - /* Count number of each code length */ - i = asize; - ci = clen; - do - { - /* Caller _must_ check that values are in-range. Most of the time the - * caller decodes a specific number of bits, which imply the max value, - * and the other time the caller decodes a huffman value, which must be - * in-range. Therefore, its an assertion and this function cannot - * otherwise fail. */ - XD3_ASSERT (*ci <= abs_max); - - nr_clen[*ci++]++; - } - while (--i != 0); - - /* Compute min, max. */ - for (i = 1; i <= abs_max; i += 1) { if (nr_clen[i]) { break; } } - min_clen = i; - for (i = abs_max; i != 0; i -= 1) { if (nr_clen[i]) { break; } } - max_clen = i; - - /* Fill the BASE, LIMIT table. */ - tmp_base[min_clen] = 0; - base[min_clen] = 0; - limit[min_clen] = nr_clen[min_clen] - 1; - for (i = min_clen + 1; i <= max_clen; i += 1) - { - usize_t last_limit = ((limit[i-1] + 1) << 1); - tmp_base[i] = tmp_base[i-1] + nr_clen[i-1]; - limit[i] = last_limit + nr_clen[i] - 1; - base[i] = last_limit - tmp_base[i]; - } - - /* Fill the inorder array, canonically ordered codes. */ - ci = clen; - for (i = 0; i < asize; i += 1) - { - if ((l = *ci++) != 0) - { - inorder[tmp_base[l]++] = i; - } - } - - *min_clenp = min_clen; - *max_clenp = max_clen; -} - -static inline int -djw_decode_symbol (xd3_stream *stream, - bit_state *bstate, - const uint8_t **input, - const uint8_t *input_end, - const uint8_t *inorder, - const usize_t *base, - const usize_t *limit, - usize_t min_clen, - usize_t max_clen, - usize_t *sym, - usize_t max_sym) -{ - usize_t code = 0; - usize_t bits = 0; - - /* OPT: Supposedly a small lookup table improves speed here... */ - - /* Code outline is similar to xd3_decode_bits... */ - if (bstate->cur_mask == 0x100) { goto next_byte; } - - for (;;) - { - do - { - if (bits == max_clen) { goto corrupt; } - - bits += 1; - code = (code << 1); - - if (bstate->cur_byte & bstate->cur_mask) { code |= 1; } - - bstate->cur_mask <<= 1; - - if (bits >= min_clen && code <= limit[bits]) { goto done; } - } - while (bstate->cur_mask != 0x100); - - next_byte: - - if (*input == input_end) - { - stream->msg = "secondary decoder end of input"; - return XD3_INTERNAL; - } - - bstate->cur_byte = *(*input)++; - bstate->cur_mask = 1; - } - - done: - - if (base[bits] <= code) - { - usize_t offset = code - base[bits]; - - if (offset <= max_sym) - { - IF_DEBUG2 (DP(RINT "(j) %u ", code)); - *sym = inorder[offset]; - return 0; - } - } - - corrupt: - stream->msg = "secondary decoder invalid code"; - return XD3_INTERNAL; -} - -static int -djw_decode_clclen (xd3_stream *stream, - bit_state *bstate, - const uint8_t **input, - const uint8_t *input_end, - uint8_t *cl_inorder, - usize_t *cl_base, - usize_t *cl_limit, - usize_t *cl_minlen, - usize_t *cl_maxlen, - uint8_t *cl_mtf) -{ - int ret; - uint8_t cl_clen[DJW_TOTAL_CODES]; - usize_t num_codes, value; - int i; - - /* How many extra code lengths to encode. */ - if ((ret = xd3_decode_bits (stream, bstate, input, - input_end, DJW_EXTRA_CODE_BITS, & num_codes))) - { - return ret; - } - - num_codes += DJW_EXTRA_12OFFSET; - - /* Read num_codes. */ - for (i = 0; i < num_codes; i += 1) - { - if ((ret = xd3_decode_bits (stream, bstate, input, - input_end, DJW_CLCLEN_BITS, & value))) - { - return ret; - } - - cl_clen[i] = value; - } - - /* Set the rest to zero. */ - for (; i < DJW_TOTAL_CODES; i += 1) { cl_clen[i] = 0; } - - /* No need to check for in-range clen values, because: */ - XD3_ASSERT (1 << DJW_CLCLEN_BITS == DJW_MAX_CLCLEN + 1); - - /* Build the code-length decoder. */ - djw_build_decoder (stream, DJW_TOTAL_CODES, DJW_MAX_CLCLEN, - cl_clen, cl_inorder, cl_base, - cl_limit, cl_minlen, cl_maxlen); - - /* Initialize the MTF state. */ - djw_init_clen_mtf_1_2 (cl_mtf); - - return 0; -} - -static inline int -djw_decode_1_2 (xd3_stream *stream, - bit_state *bstate, - const uint8_t **input, - const uint8_t *input_end, - const uint8_t *inorder, - const usize_t *base, - const usize_t *limit, - const usize_t *minlen, - const usize_t *maxlen, - uint8_t *mtfvals, - usize_t elts, - usize_t skip_offset, - uint8_t *values) -{ - usize_t n = 0, rep = 0, mtf = 0, s = 0; - int ret; - - while (n < elts) - { - /* Special case inside generic code: CLEN only: If not the first group, - * we already know the zero frequencies. */ - if (skip_offset != 0 && n >= skip_offset && values[n-skip_offset] == 0) - { - values[n++] = 0; - continue; - } - - /* Repeat last symbol. */ - if (rep != 0) - { - values[n++] = mtfvals[0]; - rep -= 1; - continue; - } - - /* Symbol following last repeat code. */ - if (mtf != 0) - { - usize_t sym = djw_update_mtf (mtfvals, mtf); - values[n++] = sym; - mtf = 0; - continue; - } - - /* Decode next symbol/repeat code. */ - if ((ret = djw_decode_symbol (stream, bstate, input, input_end, - inorder, base, limit, *minlen, *maxlen, - & mtf, DJW_TOTAL_CODES))) { return ret; } - - if (mtf <= RUN_1) - { - /* Repetition. */ - rep = ((mtf + 1) << s); - mtf = 0; - s += 1; - } - else - { - /* Remove the RUN_1 MTF offset. */ - mtf -= 1; - s = 0; - } - } - - /* If (rep != 0) there were too many codes received. */ - if (rep != 0) - { - stream->msg = "secondary decoder invalid repeat code"; - return XD3_INTERNAL; - } - - return 0; -} - -static inline int -djw_decode_prefix (xd3_stream *stream, - bit_state *bstate, - const uint8_t **input, - const uint8_t *input_end, - const uint8_t *cl_inorder, - const usize_t *cl_base, - const usize_t *cl_limit, - const usize_t *cl_minlen, - const usize_t *cl_maxlen, - uint8_t *cl_mtf, - usize_t groups, - uint8_t *clen) -{ - return djw_decode_1_2 (stream, bstate, input, input_end, - cl_inorder, cl_base, cl_limit, - cl_minlen, cl_maxlen, cl_mtf, - ALPHABET_SIZE * groups, ALPHABET_SIZE, clen); -} - -static int -xd3_decode_huff (xd3_stream *stream, - djw_stream *h, - const uint8_t **input_pos, - const uint8_t *const input_end, - uint8_t **output_pos, - const uint8_t *const output_end) -{ - const uint8_t *input = *input_pos; - uint8_t *output = *output_pos; - bit_state bstate = BIT_STATE_DECODE_INIT; - uint8_t *sel_group = NULL; - usize_t groups, gp; - usize_t output_bytes = (output_end - output); - usize_t sector_size; - usize_t sectors; - int ret; - - /* Invalid input. */ - if (output_bytes == 0) - { - stream->msg = "secondary decoder invalid input"; - return XD3_INTERNAL; - } - - /* Decode: number of groups */ - if ((ret = xd3_decode_bits (stream, & bstate, & input, - input_end, DJW_GROUP_BITS, & groups))) - { - goto fail; - } - - groups += 1; - - if (groups > 1) - { - /* Decode: group size */ - if ((ret = xd3_decode_bits (stream, & bstate, & input, - input_end, DJW_SECTORSZ_BITS, - & sector_size))) { goto fail; } - - sector_size = (sector_size + 1) * DJW_SECTORSZ_MULT; - } - else - { - /* Default for groups == 1 */ - sector_size = output_bytes; - } - - sectors = 1 + (output_bytes - 1) / sector_size; - - /* TODO: In the case of groups==1, lots of extra stack space gets used here. - * Could dynamically allocate this memory, which would help with excess - * parameter passing, too. Passing too many parameters in this file, - * simplify it! */ - - /* Outer scope: per-group symbol decoder tables. */ - { - uint8_t inorder[DJW_MAX_GROUPS][ALPHABET_SIZE]; - usize_t base [DJW_MAX_GROUPS][DJW_TOTAL_CODES]; - usize_t limit [DJW_MAX_GROUPS][DJW_TOTAL_CODES]; - usize_t minlen [DJW_MAX_GROUPS]; - usize_t maxlen [DJW_MAX_GROUPS]; - - /* Nested scope: code length decoder tables. */ - { - uint8_t clen [DJW_MAX_GROUPS][ALPHABET_SIZE]; - uint8_t cl_inorder[DJW_TOTAL_CODES]; - usize_t cl_base [DJW_MAX_CLCLEN+2]; - usize_t cl_limit [DJW_MAX_CLCLEN+2]; - uint8_t cl_mtf [DJW_TOTAL_CODES]; - usize_t cl_minlen; - usize_t cl_maxlen; - - /* Compute the code length decoder. */ - if ((ret = djw_decode_clclen (stream, & bstate, & input, input_end, - cl_inorder, cl_base, cl_limit, & cl_minlen, - & cl_maxlen, cl_mtf))) { goto fail; } - - /* Now decode each group decoder. */ - if ((ret = djw_decode_prefix (stream, & bstate, & input, input_end, - cl_inorder, cl_base, cl_limit, - & cl_minlen, & cl_maxlen, cl_mtf, - groups, clen[0]))) { goto fail; } - - /* Prepare the actual decoding tables. */ - for (gp = 0; gp < groups; gp += 1) - { - djw_build_decoder (stream, ALPHABET_SIZE, DJW_MAX_CODELEN, - clen[gp], inorder[gp], base[gp], limit[gp], - & minlen[gp], & maxlen[gp]); - } - } - - /* Decode: selector clens. */ - { - uint8_t sel_inorder[DJW_MAX_GROUPS+2]; - usize_t sel_base [DJW_MAX_GBCLEN+2]; - usize_t sel_limit [DJW_MAX_GBCLEN+2]; - uint8_t sel_mtf [DJW_MAX_GROUPS+2]; - usize_t sel_minlen; - usize_t sel_maxlen; - - /* Setup group selection. */ - if (groups > 1) - { - uint8_t sel_clen[DJW_MAX_GROUPS+1]; - - for (gp = 0; gp < groups+1; gp += 1) - { - usize_t value; - - if ((ret = xd3_decode_bits (stream, & bstate, & input, - input_end, DJW_GBCLEN_BITS, - & value))) { goto fail; } - - sel_clen[gp] = value; - sel_mtf[gp] = gp; - } - - if ((sel_group = xd3_alloc (stream, sectors, 1)) == NULL) - { - ret = ENOMEM; - goto fail; - } - - djw_build_decoder (stream, groups+1, DJW_MAX_GBCLEN, sel_clen, - sel_inorder, sel_base, sel_limit, - & sel_minlen, & sel_maxlen); - - if ((ret = djw_decode_1_2 (stream, & bstate, & input, input_end, - sel_inorder, sel_base, - sel_limit, & sel_minlen, - & sel_maxlen, sel_mtf, - sectors, 0, sel_group))) { goto fail; } - } - - /* Now decode each sector. */ - { - /* Initialize for (groups==1) case. */ - uint8_t *gp_inorder = inorder[0]; - usize_t *gp_base = base[0]; - usize_t *gp_limit = limit[0]; - usize_t gp_minlen = minlen[0]; - usize_t gp_maxlen = maxlen[0]; - usize_t c; - - for (c = 0; c < sectors; c += 1) - { - usize_t n; - - if (groups >= 2) - { - gp = sel_group[c]; - - XD3_ASSERT (gp < groups); - - gp_inorder = inorder[gp]; - gp_base = base[gp]; - gp_limit = limit[gp]; - gp_minlen = minlen[gp]; - gp_maxlen = maxlen[gp]; - } - - XD3_ASSERT (output_end - output > 0); - - /* Decode next sector. */ - n = min (sector_size, (usize_t) (output_end - output)); - - do - { - usize_t sym; - - if ((ret = djw_decode_symbol (stream, & bstate, - & input, input_end, - gp_inorder, gp_base, - gp_limit, gp_minlen, gp_maxlen, - & sym, ALPHABET_SIZE))) - { - goto fail; - } - - *output++ = sym; - } - while (--n); - } - } - } - } - - IF_REGRESSION (if ((ret = xd3_test_clean_bits (stream, & bstate))) - { goto fail; }); - XD3_ASSERT (ret == 0); - - fail: - xd3_free (stream, sel_group); - - (*input_pos) = input; - (*output_pos) = output; - return ret; -} - -#endif diff --git a/xdelta3-fgk.h b/xdelta3-fgk.h deleted file mode 100644 index d1f9b24..0000000 --- a/xdelta3-fgk.h +++ /dev/null @@ -1,852 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2002, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* For demonstration purposes only. - */ - -#ifndef _XDELTA3_FGK_h_ -#define _XDELTA3_FGK_h_ - -/* An implementation of the FGK algorithm described by D.E. Knuth in - * "Dynamic Huffman Coding" in Journal of Algorithms 6. */ - -/* A 32bit counter (fgk_weight) is used as the frequency counter for - * nodes in the huffman tree. TODO: Need oto test for overflow and/or - * reset stats. */ - -typedef struct _fgk_stream fgk_stream; -typedef struct _fgk_node fgk_node; -typedef struct _fgk_block fgk_block; -typedef unsigned int fgk_bit; -typedef uint32_t fgk_weight; - -struct _fgk_block { - union { - fgk_node *un_leader; - fgk_block *un_freeptr; - } un; -}; - -#define block_leader un.un_leader -#define block_freeptr un.un_freeptr - -/* The code can also support fixed huffman encoding/decoding. */ -#define IS_ADAPTIVE 1 - -/* weight is a count of the number of times this element has been seen - * in the current encoding/decoding. parent, right_child, and - * left_child are pointers defining the tree structure. right and - * left point to neighbors in an ordered sequence of weights. The - * left child of a node is always guaranteed to have weight not - * greater than its sibling. fgk_blockLeader points to the element - * with the same weight as itself which is closest to the next - * increasing weight block. */ -struct _fgk_node -{ - fgk_weight weight; - fgk_node *parent; - fgk_node *left_child; - fgk_node *right_child; - fgk_node *left; - fgk_node *right; - fgk_block *my_block; -}; - -/* alphabet_size is the a count of the number of possible leaves in - * the huffman tree. The number of total nodes counting internal - * nodes is ((2 * alphabet_size) - 1). zero_freq_count is the number - * of elements remaining which have zero frequency. zero_freq_exp and - * zero_freq_rem satisfy the equation zero_freq_count = - * 2^zero_freq_exp + zero_freq_rem. root_node is the root of the - * tree, which is initialized to a node with zero frequency and - * contains the 0th such element. free_node contains a pointer to the - * next available fgk_node space. alphabet contains all the elements - * and is indexed by N. remaining_zeros points to the head of the - * list of zeros. */ -struct _fgk_stream -{ - int alphabet_size; - int zero_freq_count; - int zero_freq_exp; - int zero_freq_rem; - int coded_depth; - - int total_nodes; - int total_blocks; - - fgk_bit *coded_bits; - - fgk_block *block_array; - fgk_block *free_block; - - fgk_node *decode_ptr; - fgk_node *remaining_zeros; - fgk_node *alphabet; - fgk_node *root_node; - fgk_node *free_node; -}; - -/*********************************************************************/ -/* Encoder */ -/*********************************************************************/ - -static fgk_stream* fgk_alloc (xd3_stream *stream /*, int alphabet_size */); -static void fgk_init (fgk_stream *h); -static int fgk_encode_data (fgk_stream *h, - int n); -static inline fgk_bit fgk_get_encoded_bit (fgk_stream *h); - -static int xd3_encode_fgk (xd3_stream *stream, - fgk_stream *sec_stream, - xd3_output *input, - xd3_output *output, - xd3_sec_cfg *cfg); - -/*********************************************************************/ -/* Decoder */ -/*********************************************************************/ - -static inline int fgk_decode_bit (fgk_stream *h, - fgk_bit b); -static int fgk_decode_data (fgk_stream *h); -static void fgk_destroy (xd3_stream *stream, - fgk_stream *h); - -static int xd3_decode_fgk (xd3_stream *stream, - fgk_stream *sec_stream, - const uint8_t **input, - const uint8_t *const input_end, - uint8_t **output, - const uint8_t *const output_end); - -/*********************************************************************/ -/* Private */ -/*********************************************************************/ - -static unsigned int fgk_find_nth_zero (fgk_stream *h, int n); -static int fgk_nth_zero (fgk_stream *h, int n); -static void fgk_update_tree (fgk_stream *h, int n); -static fgk_node* fgk_increase_zero_weight (fgk_stream *h, int n); -static void fgk_eliminate_zero (fgk_stream* h, fgk_node *node); -static void fgk_move_right (fgk_stream *h, fgk_node *node); -static void fgk_promote (fgk_stream *h, fgk_node *node); -static void fgk_init_node (fgk_node *node, int i, int size); -static fgk_block* fgk_make_block (fgk_stream *h, fgk_node *l); -static void fgk_free_block (fgk_stream *h, fgk_block *b); -static void fgk_factor_remaining (fgk_stream *h); -static inline void fgk_swap_ptrs (fgk_node **one, fgk_node **two); - -/*********************************************************************/ -/* Basic Routines */ -/*********************************************************************/ - -/* returns an initialized huffman encoder for an alphabet with the - * given size. returns NULL if enough memory cannot be allocated */ -static fgk_stream* fgk_alloc (xd3_stream *stream /*, int alphabet_size0 */) -{ - int alphabet_size0 = ALPHABET_SIZE; - fgk_stream *h; - - if ((h = (fgk_stream*) xd3_alloc (stream, 1, sizeof (fgk_stream))) == NULL) - { - return NULL; - } - - h->total_nodes = (2 * alphabet_size0) - 1; - h->total_blocks = (2 * h->total_nodes); - h->alphabet = (fgk_node*) xd3_alloc (stream, h->total_nodes, sizeof (fgk_node)); - h->block_array = (fgk_block*) xd3_alloc (stream, h->total_blocks, sizeof (fgk_block)); - h->coded_bits = (fgk_bit*) xd3_alloc (stream, alphabet_size0, sizeof (fgk_bit)); - - if (h->coded_bits == NULL || - h->alphabet == NULL || - h->block_array == NULL) - { - fgk_destroy (stream, h); - return NULL; - } - - h->alphabet_size = alphabet_size0; - - return h; -} - -static void fgk_init (fgk_stream *h) -{ - int i; - - h->root_node = h->alphabet; - h->decode_ptr = h->root_node; - h->free_node = h->alphabet + h->alphabet_size; - h->remaining_zeros = h->alphabet; - h->coded_depth = 0; - h->zero_freq_count = h->alphabet_size + 2; - - /* after two calls to factor_remaining, zero_freq_count == alphabet_size */ - fgk_factor_remaining(h); /* set ZFE and ZFR */ - fgk_factor_remaining(h); /* set ZFDB according to prev state */ - - IF_DEBUG (memset (h->alphabet, 0, sizeof (h->alphabet[0]) * h->total_nodes)); - - for (i = 0; i < h->total_blocks-1; i += 1) - { - h->block_array[i].block_freeptr = &h->block_array[i + 1]; - } - - h->block_array[h->total_blocks - 1].block_freeptr = NULL; - h->free_block = h->block_array; - - /* Zero frequency nodes are inserted in the first alphabet_size - * positions, with Value, weight, and a pointer to the next zero - * frequency node. */ - for (i = h->alphabet_size - 1; i >= 0; i -= 1) - { - fgk_init_node (h->alphabet + i, i, h->alphabet_size); - } -} - -static void fgk_swap_ptrs(fgk_node **one, fgk_node **two) -{ - fgk_node *tmp = *one; - *one = *two; - *two = tmp; -} - -/* Takes huffman transmitter h and n, the nth elt in the alphabet, and - * returns the number of required to encode n. */ -static int fgk_encode_data (fgk_stream* h, int n) -{ - fgk_node *target_ptr = h->alphabet + n; - - XD3_ASSERT (n < h->alphabet_size); - - h->coded_depth = 0; - - /* First encode the binary representation of the nth remaining - * zero frequency element in reverse such that bit, which will be - * encoded from h->coded_depth down to 0 will arrive in increasing - * order following the tree path. If there is only one left, it - * is not neccesary to encode these bits. */ - if (IS_ADAPTIVE && target_ptr->weight == 0) - { - unsigned int where, shift; - int bits; - - where = fgk_find_nth_zero(h, n); - shift = 1; - - if (h->zero_freq_rem == 0) - { - bits = h->zero_freq_exp; - } - else - { - bits = h->zero_freq_exp + 1; - } - - while (bits > 0) - { - h->coded_bits[h->coded_depth++] = (shift & where) && 1; - - bits -= 1; - shift <<= 1; - }; - - target_ptr = h->remaining_zeros; - } - - /* The path from root to node is filled into coded_bits in reverse so - * that it is encoded in the right order */ - while (target_ptr != h->root_node) - { - h->coded_bits[h->coded_depth++] = (target_ptr->parent->right_child == target_ptr); - - target_ptr = target_ptr->parent; - } - - if (IS_ADAPTIVE) - { - fgk_update_tree(h, n); - } - - return h->coded_depth; -} - -/* Should be called as many times as fgk_encode_data returns. - */ -static inline fgk_bit fgk_get_encoded_bit (fgk_stream *h) -{ - XD3_ASSERT (h->coded_depth > 0); - - return h->coded_bits[--h->coded_depth]; -} - -/* This procedure updates the tree after alphabet[n] has been encoded - * or decoded. - */ -static void fgk_update_tree (fgk_stream *h, int n) -{ - fgk_node *incr_node; - - if (h->alphabet[n].weight == 0) - { - incr_node = fgk_increase_zero_weight (h, n); - } - else - { - incr_node = h->alphabet + n; - } - - while (incr_node != h->root_node) - { - fgk_move_right (h, incr_node); - fgk_promote (h, incr_node); - incr_node->weight += 1; /* incr the parent */ - incr_node = incr_node->parent; /* repeat */ - } - - h->root_node->weight += 1; -} - -static void fgk_move_right (fgk_stream *h, fgk_node *move_fwd) -{ - fgk_node **fwd_par_ptr, **back_par_ptr; - fgk_node *move_back, *tmp; - - move_back = move_fwd->my_block->block_leader; - - if (move_fwd == move_back || - move_fwd->parent == move_back || - move_fwd->weight == 0) - { - return; - } - - move_back->right->left = move_fwd; - - if (move_fwd->left) - { - move_fwd->left->right = move_back; - } - - tmp = move_fwd->right; - move_fwd->right = move_back->right; - - if (tmp == move_back) - { - move_back->right = move_fwd; - } - else - { - tmp->left = move_back; - move_back->right = tmp; - } - - tmp = move_back->left; - move_back->left = move_fwd->left; - - if (tmp == move_fwd) - { - move_fwd->left = move_back; - } - else - { - tmp->right = move_fwd; - move_fwd->left = tmp; - } - - if (move_fwd->parent->right_child == move_fwd) - { - fwd_par_ptr = &move_fwd->parent->right_child; - } - else - { - fwd_par_ptr = &move_fwd->parent->left_child; - } - - if (move_back->parent->right_child == move_back) - { - back_par_ptr = &move_back->parent->right_child; - } - else - { - back_par_ptr = &move_back->parent->left_child; - } - - fgk_swap_ptrs (&move_fwd->parent, &move_back->parent); - fgk_swap_ptrs (fwd_par_ptr, back_par_ptr); - - move_fwd->my_block->block_leader = move_fwd; -} - -/* Shifts node, the leader of its block, into the next block. */ -static void fgk_promote (fgk_stream *h, fgk_node *node) -{ - fgk_node *my_left, *my_right; - fgk_block *cur_block; - - my_right = node->right; - my_left = node->left; - cur_block = node->my_block; - - if (node->weight == 0) - { - return; - } - - /* if left is right child, parent of remaining zeros case (?), means parent - * has same weight as right child. */ - if (my_left == node->right_child && - node->left_child && - node->left_child->weight == 0) - { - XD3_ASSERT (node->left_child == h->remaining_zeros); - XD3_ASSERT (node->right_child->weight == (node->weight+1)); /* child weight was already incremented */ - - if (node->weight == (my_right->weight - 1) && my_right != h->root_node) - { - fgk_free_block (h, cur_block); - node->my_block = my_right->my_block; - my_left->my_block = my_right->my_block; - } - - return; - } - - if (my_left == h->remaining_zeros) - { - return; - } - - /* true if not the leftmost node */ - if (my_left->my_block == cur_block) - { - my_left->my_block->block_leader = my_left; - } - else - { - fgk_free_block (h, cur_block); - } - - /* node->parent != my_right */ - if ((node->weight == (my_right->weight - 1)) && (my_right != h->root_node)) - { - node->my_block = my_right->my_block; - } - else - { - node->my_block = fgk_make_block (h, node); - } -} - -/* When an element is seen the first time this is called to remove it from the list of - * zero weight elements and introduce a new internal node to the tree. */ -static fgk_node* fgk_increase_zero_weight (fgk_stream *h, int n) -{ - fgk_node *this_zero, *new_internal, *zero_ptr; - - this_zero = h->alphabet + n; - - if (h->zero_freq_count == 1) - { - /* this is the last one */ - this_zero->right_child = NULL; - - if (this_zero->right->weight == 1) - { - this_zero->my_block = this_zero->right->my_block; - } - else - { - this_zero->my_block = fgk_make_block (h, this_zero); - } - - h->remaining_zeros = NULL; - - return this_zero; - } - - zero_ptr = h->remaining_zeros; - - new_internal = h->free_node++; - - new_internal->parent = zero_ptr->parent; - new_internal->right = zero_ptr->right; - new_internal->weight = 0; - new_internal->right_child = this_zero; - new_internal->left = this_zero; - - if (h->remaining_zeros == h->root_node) - { - /* This is the first element to be coded */ - h->root_node = new_internal; - this_zero->my_block = fgk_make_block (h, this_zero); - new_internal->my_block = fgk_make_block (h, new_internal); - } - else - { - new_internal->right->left = new_internal; - - if (zero_ptr->parent->right_child == zero_ptr) - { - zero_ptr->parent->right_child = new_internal; - } - else - { - zero_ptr->parent->left_child = new_internal; - } - - if (new_internal->right->weight == 1) - { - new_internal->my_block = new_internal->right->my_block; - } - else - { - new_internal->my_block = fgk_make_block (h, new_internal); - } - - this_zero->my_block = new_internal->my_block; - } - - fgk_eliminate_zero (h, this_zero); - - new_internal->left_child = h->remaining_zeros; - - this_zero->right = new_internal; - this_zero->left = h->remaining_zeros; - this_zero->parent = new_internal; - this_zero->left_child = NULL; - this_zero->right_child = NULL; - - h->remaining_zeros->parent = new_internal; - h->remaining_zeros->right = this_zero; - - return this_zero; -} - -/* When a zero frequency element is encoded, it is followed by the - * binary representation of the index into the remaining elements. - * Sets a cache to the element before it so that it can be removed - * without calling this procedure again. */ -static unsigned int fgk_find_nth_zero (fgk_stream* h, int n) -{ - fgk_node *target_ptr = h->alphabet + n; - fgk_node *head_ptr = h->remaining_zeros; - unsigned int idx = 0; - - while (target_ptr != head_ptr) - { - head_ptr = head_ptr->right_child; - idx += 1; - } - - return idx; -} - -/* Splices node out of the list of zeros. */ -static void fgk_eliminate_zero (fgk_stream* h, fgk_node *node) -{ - if (h->zero_freq_count == 1) - { - return; - } - - fgk_factor_remaining(h); - - if (node->left_child == NULL) - { - h->remaining_zeros = h->remaining_zeros->right_child; - h->remaining_zeros->left_child = NULL; - } - else if (node->right_child == NULL) - { - node->left_child->right_child = NULL; - } - else - { - node->right_child->left_child = node->left_child; - node->left_child->right_child = node->right_child; - } -} - -static void fgk_init_node (fgk_node *node, int i, int size) -{ - if (i < size - 1) - { - node->right_child = node + 1; - } - else - { - node->right_child = NULL; - } - - if (i >= 1) - { - node->left_child = node - 1; - } - else - { - node->left_child = NULL; - } - - node->weight = 0; - node->parent = NULL; - node->right = NULL; - node->left = NULL; - node->my_block = NULL; -} - -/* The data structure used is an array of blocks, which are unions of - * free pointers and huffnode pointers. free blocks are a linked list - * of free blocks, the front of which is h->free_block. The used - * blocks are pointers to the head of each block. */ -static fgk_block* fgk_make_block (fgk_stream *h, fgk_node* lead) -{ - fgk_block *ret = h->free_block; - - XD3_ASSERT (h->free_block != NULL); - - h->free_block = h->free_block->block_freeptr; - - ret->block_leader = lead; - - return ret; -} - -/* Restores the block to the front of the free list. */ -static void fgk_free_block (fgk_stream *h, fgk_block *b) -{ - b->block_freeptr = h->free_block; - h->free_block = b; -} - -/* sets zero_freq_count, zero_freq_rem, and zero_freq_exp to satsity - * the equation given above. */ -static void fgk_factor_remaining (fgk_stream *h) -{ - unsigned int i; - - i = (--h->zero_freq_count); - h->zero_freq_exp = 0; - - while (i > 1) - { - h->zero_freq_exp += 1; - i >>= 1; - } - - i = 1 << h->zero_freq_exp; - - h->zero_freq_rem = h->zero_freq_count - i; -} - -/* receives a bit at a time and returns true when a complete code has - * been received. - */ -static int inline fgk_decode_bit (fgk_stream* h, fgk_bit b) -{ - XD3_ASSERT (b == 1 || b == 0); - - if (IS_ADAPTIVE && h->decode_ptr->weight == 0) - { - int bitsreq; - - if (h->zero_freq_rem == 0) - { - bitsreq = h->zero_freq_exp; - } - else - { - bitsreq = h->zero_freq_exp + 1; - } - - h->coded_bits[h->coded_depth] = b; - h->coded_depth += 1; - - return h->coded_depth >= bitsreq; - } - else - { - if (b) - { - h->decode_ptr = h->decode_ptr->right_child; - } - else - { - h->decode_ptr = h->decode_ptr->left_child; - } - - if (h->decode_ptr->left_child == NULL) - { - /* If the weight is non-zero, finished. */ - if (h->decode_ptr->weight != 0) - { - return 1; - } - - /* zero_freq_count is dropping to 0, finished. */ - return h->zero_freq_count == 1; - } - else - { - return 0; - } - } -} - -static int fgk_nth_zero (fgk_stream* h, int n) -{ - fgk_node *ret = h->remaining_zeros; - - /* ERROR: if during this loop (ret->right_child == NULL) then the - * encoder's zero count is too high. Could return an error code - * now, but is probably unnecessary overhead, since the caller - * should check integrity anyway. */ - for (; n != 0 && ret->right_child != NULL; n -= 1) - { - ret = ret->right_child; - } - - return ret - h->alphabet; -} - -/* once fgk_decode_bit returns 1, this retrieves an index into the - * alphabet otherwise this returns 0, indicating more bits are - * required. - */ -static int fgk_decode_data (fgk_stream* h) -{ - unsigned int elt = h->decode_ptr - h->alphabet; - - if (IS_ADAPTIVE && h->decode_ptr->weight == 0) { - int i; - unsigned int n = 0; - - for (i = 0; i < h->coded_depth - 1; i += 1) - { - n |= h->coded_bits[i]; - n <<= 1; - } - - n |= h->coded_bits[i]; - elt = fgk_nth_zero(h, n); - } - - h->coded_depth = 0; - - if (IS_ADAPTIVE) - { - fgk_update_tree(h, elt); - } - - h->decode_ptr = h->root_node; - - return elt; -} - -static void fgk_destroy (xd3_stream *stream, - fgk_stream *h) -{ - if (h != NULL) - { - xd3_free (stream, h->alphabet); - xd3_free (stream, h->coded_bits); - xd3_free (stream, h->block_array); - xd3_free (stream, h); - } -} - -/*********************************************************************/ -/* Xdelta */ -/*********************************************************************/ - -static int -xd3_encode_fgk (xd3_stream *stream, fgk_stream *sec_stream, xd3_output *input, xd3_output *output, xd3_sec_cfg *cfg) -{ - bit_state bstate = BIT_STATE_ENCODE_INIT; - xd3_output *cur_page; - int ret; - - /* OPT: quit compression early if it looks bad */ - for (cur_page = input; cur_page; cur_page = cur_page->next_page) - { - const uint8_t *inp = cur_page->base; - const uint8_t *inp_max = inp + cur_page->next; - - while (inp < inp_max) - { - usize_t bits = fgk_encode_data (sec_stream, *inp++); - - while (bits--) - { - if ((ret = xd3_encode_bit (stream, & output, & bstate, fgk_get_encoded_bit (sec_stream)))) { return ret; } - } - } - } - - return xd3_flush_bits (stream, & output, & bstate); -} - -static int -xd3_decode_fgk (xd3_stream *stream, - fgk_stream *sec_stream, - const uint8_t **input_pos, - const uint8_t *const input_max, - uint8_t **output_pos, - const uint8_t *const output_max) -{ - bit_state bstate; - uint8_t *output = *output_pos; - const uint8_t *input = *input_pos; - - for (;;) - { - if (input == input_max) - { - stream->msg = "secondary decoder end of input"; - return XD3_INTERNAL; - } - - bstate.cur_byte = *input++; - - for (bstate.cur_mask = 1; bstate.cur_mask != 0x100; bstate.cur_mask <<= 1) - { - int done = fgk_decode_bit (sec_stream, (bstate.cur_byte & bstate.cur_mask) && 1); - - if (! done) { continue; } - - *output++ = fgk_decode_data (sec_stream); - - if (output == output_max) - { - /* During regression testing: */ - IF_REGRESSION ({ - int ret; - bstate.cur_mask <<= 1; - if ((ret = xd3_test_clean_bits (stream, & bstate))) { return ret; } - }); - - (*output_pos) = output; - (*input_pos) = input; - return 0; - } - } - } -} - -#endif /* _XDELTA3_FGK_ */ diff --git a/xdelta3-hash.h b/xdelta3-hash.h deleted file mode 100644 index b098d24..0000000 --- a/xdelta3-hash.h +++ /dev/null @@ -1,223 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _XDELTA3_HASH_H_ -#define _XDELTA3_HASH_H_ - -#if XD3_DEBUG -#define SMALL_HASH_DEBUG1(s,inp) \ - usize_t debug_state; \ - usize_t debug_hval = xd3_checksum_hash (& (s)->small_hash, \ - xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look)) -#define SMALL_HASH_DEBUG2(s,inp) \ - XD3_ASSERT (debug_hval == xd3_checksum_hash (& (s)->small_hash, \ - xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look))) -#else -#define SMALL_HASH_DEBUG1(s,inp) -#define SMALL_HASH_DEBUG2(s,inp) -#endif /* XD3_DEBUG */ - -/* This is a good hash multiplier for 32-bit LCGs: see "linear - * congruential generators of different sizes and good lattice - * structure" */ -static const uint32_t hash_multiplier = 1597334677U; - -/*********************************************************************** - Permute stuff - ***********************************************************************/ - -#if HASH_PERMUTE == 0 -#define PERMUTE(x) (x) -#else -#define PERMUTE(x) (__single_hash[(uint32_t)x]) - -static const uint16_t __single_hash[256] = -{ - /* Random numbers generated using SLIB's pseudo-random number generator. - * This hashes the input alphabet. */ - 0xbcd1, 0xbb65, 0x42c2, 0xdffe, 0x9666, 0x431b, 0x8504, 0xeb46, - 0x6379, 0xd460, 0xcf14, 0x53cf, 0xdb51, 0xdb08, 0x12c8, 0xf602, - 0xe766, 0x2394, 0x250d, 0xdcbb, 0xa678, 0x02af, 0xa5c6, 0x7ea6, - 0xb645, 0xcb4d, 0xc44b, 0xe5dc, 0x9fe6, 0x5b5c, 0x35f5, 0x701a, - 0x220f, 0x6c38, 0x1a56, 0x4ca3, 0xffc6, 0xb152, 0x8d61, 0x7a58, - 0x9025, 0x8b3d, 0xbf0f, 0x95a3, 0xe5f4, 0xc127, 0x3bed, 0x320b, - 0xb7f3, 0x6054, 0x333c, 0xd383, 0x8154, 0x5242, 0x4e0d, 0x0a94, - 0x7028, 0x8689, 0x3a22, 0x0980, 0x1847, 0xb0f1, 0x9b5c, 0x4176, - 0xb858, 0xd542, 0x1f6c, 0x2497, 0x6a5a, 0x9fa9, 0x8c5a, 0x7743, - 0xa8a9, 0x9a02, 0x4918, 0x438c, 0xc388, 0x9e2b, 0x4cad, 0x01b6, - 0xab19, 0xf777, 0x365f, 0x1eb2, 0x091e, 0x7bf8, 0x7a8e, 0x5227, - 0xeab1, 0x2074, 0x4523, 0xe781, 0x01a3, 0x163d, 0x3b2e, 0x287d, - 0x5e7f, 0xa063, 0xb134, 0x8fae, 0x5e8e, 0xb7b7, 0x4548, 0x1f5a, - 0xfa56, 0x7a24, 0x900f, 0x42dc, 0xcc69, 0x02a0, 0x0b22, 0xdb31, - 0x71fe, 0x0c7d, 0x1732, 0x1159, 0xcb09, 0xe1d2, 0x1351, 0x52e9, - 0xf536, 0x5a4f, 0xc316, 0x6bf9, 0x8994, 0xb774, 0x5f3e, 0xf6d6, - 0x3a61, 0xf82c, 0xcc22, 0x9d06, 0x299c, 0x09e5, 0x1eec, 0x514f, - 0x8d53, 0xa650, 0x5c6e, 0xc577, 0x7958, 0x71ac, 0x8916, 0x9b4f, - 0x2c09, 0x5211, 0xf6d8, 0xcaaa, 0xf7ef, 0x287f, 0x7a94, 0xab49, - 0xfa2c, 0x7222, 0xe457, 0xd71a, 0x00c3, 0x1a76, 0xe98c, 0xc037, - 0x8208, 0x5c2d, 0xdfda, 0xe5f5, 0x0b45, 0x15ce, 0x8a7e, 0xfcad, - 0xaa2d, 0x4b5c, 0xd42e, 0xb251, 0x907e, 0x9a47, 0xc9a6, 0xd93f, - 0x085e, 0x35ce, 0xa153, 0x7e7b, 0x9f0b, 0x25aa, 0x5d9f, 0xc04d, - 0x8a0e, 0x2875, 0x4a1c, 0x295f, 0x1393, 0xf760, 0x9178, 0x0f5b, - 0xfa7d, 0x83b4, 0x2082, 0x721d, 0x6462, 0x0368, 0x67e2, 0x8624, - 0x194d, 0x22f6, 0x78fb, 0x6791, 0xb238, 0xb332, 0x7276, 0xf272, - 0x47ec, 0x4504, 0xa961, 0x9fc8, 0x3fdc, 0xb413, 0x007a, 0x0806, - 0x7458, 0x95c6, 0xccaa, 0x18d6, 0xe2ae, 0x1b06, 0xf3f6, 0x5050, - 0xc8e8, 0xf4ac, 0xc04c, 0xf41c, 0x992f, 0xae44, 0x5f1b, 0x1113, - 0x1738, 0xd9a8, 0x19ea, 0x2d33, 0x9698, 0x2fe9, 0x323f, 0xcde2, - 0x6d71, 0xe37d, 0xb697, 0x2c4f, 0x4373, 0x9102, 0x075d, 0x8e25, - 0x1672, 0xec28, 0x6acb, 0x86cc, 0x186e, 0x9414, 0xd674, 0xd1a5 -}; -#endif - -/* Update the checksum state. */ -#if ADLER_LARGE_CKSUM -inline uint32_t -xd3_large_cksum_update (uint32_t cksum, - const uint8_t *base, - int look) { - uint32_t old_c = PERMUTE(base[0]); - uint32_t new_c = PERMUTE(base[look]); - uint32_t low = ((cksum & 0xffff) - old_c + new_c) & 0xffff; - uint32_t high = ((cksum >> 16) - (old_c * look) + low) & 0xffff; - return (high << 16) | low; -} -#else -// TODO: revisit this topic -#endif - -/* Note: small cksum is hard-coded for 4 bytes */ -#if UNALIGNED_OK -static inline uint32_t -xd3_scksum (uint32_t *state, - const uint8_t *base, - const int look) -{ - (*state) = *(uint32_t*)base; - return (*state) * hash_multiplier; -} -static inline uint32_t -xd3_small_cksum_update (uint32_t *state, - const uint8_t *base, - int look) -{ - (*state) = *(uint32_t*)(base+1); - return (*state) * hash_multiplier; -} -#else -static inline uint32_t -xd3_scksum (uint32_t *state, - const uint8_t *base, - const int look) -{ - (*state) = (base[0] << 24 | - base[1] << 16 | - base[2] << 8 | - base[3]); - return (*state) * hash_multiplier; -} -static inline uint32_t -xd3_small_cksum_update (uint32_t *state, - const uint8_t *base, - const int look) -{ - (*state) <<= 8; - (*state) |= base[4]; - return (*state) * hash_multiplier; -} -#endif - -/*********************************************************************** - Ctable stuff - ***********************************************************************/ - -static inline usize_t -xd3_checksum_hash (const xd3_hash_cfg *cfg, const usize_t cksum) -{ - return (cksum >> cfg->shift) ^ (cksum & cfg->mask); -} - -/*********************************************************************** - Cksum function - ***********************************************************************/ - -#if ADLER_LARGE_CKSUM -static inline uint32_t -xd3_lcksum (const uint8_t *seg, const int ln) -{ - int i = 0; - uint32_t low = 0; - uint32_t high = 0; - - for (; i < ln; i += 1) - { - low += PERMUTE(*seg++); - high += low; - } - - return ((high & 0xffff) << 16) | (low & 0xffff); -} -#else -static inline uint32_t -xd3_lcksum (const uint8_t *seg, const int ln) -{ - int i, j; - uint32_t h = 0; - for (i = 0, j = ln - 1; i < ln; ++i, --j) { - h += PERMUTE(seg[i]) * hash_multiplier_powers[j]; - } - return h; -} -#endif - -#if XD3_ENCODER -static usize_t -xd3_size_log2 (usize_t slots) -{ - int bits = 28; /* This should not be an unreasonable limit. */ - int i; - - for (i = 3; i <= bits; i += 1) - { - if (slots < (1U << i)) - { - /* TODO: this is compaction=1 in checksum_test.cc and maybe should - * not be fixed at -1. */ - bits = i - 1; - break; - } - } - - return bits; -} - -static void -xd3_size_hashtable (xd3_stream *stream, - usize_t slots, - xd3_hash_cfg *cfg) -{ - int bits = xd3_size_log2 (slots); - - /* TODO: there's a 32-bit assumption here */ - cfg->size = (1 << bits); - cfg->mask = (cfg->size - 1); - cfg->shift = 32 - bits; -} -#endif - -#endif diff --git a/xdelta3-list.h b/xdelta3-list.h deleted file mode 100644 index 3c0df5e..0000000 --- a/xdelta3-list.h +++ /dev/null @@ -1,130 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2002, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef __XDELTA3_LIST__ -#define __XDELTA3_LIST__ - -#define XD3_MAKELIST(LTYPE,ETYPE,LNAME) \ - \ -static inline ETYPE* \ -LTYPE ## _entry (LTYPE* l) \ -{ \ - return (ETYPE*) ((char*) l - (unsigned long) &((ETYPE*) 0)->LNAME); \ -} \ - \ -static inline void \ -LTYPE ## _init (LTYPE *l) \ -{ \ - l->next = l; \ - l->prev = l; \ -} \ - \ -static inline void \ -LTYPE ## _add (LTYPE *prev, LTYPE *next, LTYPE *ins) \ -{ \ - next->prev = ins; \ - prev->next = ins; \ - ins->next = next; \ - ins->prev = prev; \ -} \ - \ -static inline void \ -LTYPE ## _push_back (LTYPE *l, ETYPE *i) \ -{ \ - LTYPE ## _add (l->prev, l, & i->LNAME); \ -} \ - \ -static inline void \ -LTYPE ## _del (LTYPE *next, \ - LTYPE *prev) \ -{ \ - next->prev = prev; \ - prev->next = next; \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _remove (ETYPE *f) \ -{ \ - LTYPE *i = f->LNAME.next; \ - LTYPE ## _del (f->LNAME.next, f->LNAME.prev); \ - return LTYPE ## _entry (i); \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _pop_back (LTYPE *l) \ -{ \ - LTYPE *i = l->prev; \ - LTYPE ## _del (i->next, i->prev); \ - return LTYPE ## _entry (i); \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _pop_front (LTYPE *l) \ -{ \ - LTYPE *i = l->next; \ - LTYPE ## _del (i->next, i->prev); \ - return LTYPE ## _entry (i); \ -} \ - \ -static inline int \ -LTYPE ## _empty (LTYPE *l) \ -{ \ - return l == l->next; \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _front (LTYPE *f) \ -{ \ - return LTYPE ## _entry (f->next); \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _back (LTYPE *f) \ -{ \ - return LTYPE ## _entry (f->prev); \ -} \ - \ -static inline int \ -LTYPE ## _end (LTYPE *f, ETYPE *i) \ -{ \ - return f == & i->LNAME; \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _next (ETYPE *f) \ -{ \ - return LTYPE ## _entry (f->LNAME.next); \ -} \ - \ -static inline usize_t \ -LTYPE ## _length (LTYPE *l) \ -{ \ - LTYPE *p; \ - int c = 0; \ - \ - for (p = l->next; p != l; p = p->next) \ - { \ - c += 1; \ - } \ - \ - return c; \ -} \ - \ -typedef int unused_ ## LTYPE - -#endif diff --git a/xdelta3-main.h b/xdelta3-main.h deleted file mode 100644 index 55200bd..0000000 --- a/xdelta3-main.h +++ /dev/null @@ -1,4242 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, - * Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* This is all the extra stuff you need for convenience to users in a - * command line application. It contains these major components: - * - * 1. VCDIFF tools 2. external compression support (this is - * POSIX-specific). 3. a general read/write loop that handles all of - * the Xdelta decode/encode/VCDIFF-print functions 4. command-line - * interpreter 5. an Xdelta application header which stores default - * filename, external compression settings 6. output/error printing - * 7. basic file support and OS interface - */ - -/* TODO list: 1. do exact gzip-like filename, stdout handling. make a - * .vcdiff extension, refuse to encode to stdout without -cf, etc. - * 2. Allow the user to add a comment string to the app header without - * disturbing the default behavior. 3. "Source file must be seekable" - * is not actually true for encoding, given current behavior. Allow - * non-seekable sources? It would in theory let you use a fifo for - * the source. - */ - -/* On error handling and printing: - * - * The xdelta library sets stream->msg to indicate what condition - * caused an internal failure, but many failures originate here and - * are printed here. The return convention is 0 for success, as - * throughout Xdelta code, but special attention is required here for - * the operating system calls with different error handling. See the - * main_file_* routines. All errors in this file have a message - * printed at the time of occurance. Since some of these calls occur - * within calls to the library, the error may end up being printed - * again with a more general error message. - */ - -/*********************************************************************/ - -#ifndef XD3_POSIX -#define XD3_POSIX 0 -#endif -#ifndef XD3_STDIO -#define XD3_STDIO 0 -#endif -#ifndef XD3_WIN32 -#define XD3_WIN32 0 -#endif -#ifndef NOT_MAIN -#define NOT_MAIN 0 -#endif - -/* Combines xd3_strerror() and strerror() */ -const char* xd3_mainerror(int err_num); - -/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */ -#define XPR fprintf -#define NT stderr, "xdelta3: " - -/* If none are set, default to posix. */ -#if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0 -#undef XD3_POSIX -#define XD3_POSIX 1 -#endif - -/* Handle externally-compressed inputs. */ -#ifndef EXTERNAL_COMPRESSION -#define EXTERNAL_COMPRESSION 1 -#endif - -#define PRINTHDR_SPECIAL -4378291 - -/* The number of soft-config variables. */ -#define XD3_SOFTCFG_VARCNT 7 - -/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an - * error message from the library. */ -#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", \ - xd3_errstring (stream), xd3_mainerror (ret) - -#include <stdio.h> /* fprintf */ - -#if XD3_POSIX -#include <unistd.h> /* close, read, write... */ -#include <sys/types.h> -#include <fcntl.h> -#endif - -#ifndef _WIN32 -#include <unistd.h> /* lots */ -#include <sys/time.h> /* gettimeofday() */ -#include <sys/stat.h> /* stat() and fstat() */ -#else -#if defined(_MSC_VER) -#define strtoll _strtoi64 -#endif -#include <sys/types.h> -#include <sys/stat.h> -#ifndef WIFEXITED -# define WIFEXITED(stat) (((*((int *) &(stat))) & 0xff) == 0) -#endif -#ifndef WEXITSTATUS -# define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff) -#endif -#ifndef S_ISREG -//# ifdef S_IFREG -//# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) -//# else -# define S_ISREG(m) 1 -//# endif -#endif /* !S_ISREG */ - -// For standard input/output handles -static STARTUPINFO winStartupInfo; -#endif - -/********************************************************************** - ENUMS and TYPES - *********************************************************************/ - -/* These flags (mainly pertaining to main_read() operations) are set - * in the main_file->flags variable. All are related to with external - * decompression support. - * - * RD_FIRST causes the external decompression check when the input is - * first read. - * - * RD_NONEXTERNAL disables external decompression for reading a - * compressed input, in the case of Xdelta inputs. Note: Xdelta is - * supported as an external compression type, which makes is the - * reason for this flag. An example to justify this is: to create a - * delta between two files that are VCDIFF-compressed. Two external - * Xdelta decoders are run to supply decompressed source and target - * inputs to the Xdelta encoder. */ -typedef enum -{ - RD_FIRST = (1 << 0), - RD_NONEXTERNAL = (1 << 1), - RD_EXTERNAL_V1 = (1 << 2), -} xd3_read_flags; - -/* main_file->mode values */ -typedef enum -{ - XO_READ = 0, - XO_WRITE = 1, -} main_file_modes; - -/* Main commands. For example, CMD_PRINTHDR is the "xdelta printhdr" - * command. */ -typedef enum -{ - CMD_NONE = 0, - CMD_PRINTHDR, - CMD_PRINTHDRS, - CMD_PRINTDELTA, - CMD_RECODE, - CMD_MERGE_ARG, - CMD_MERGE, -#if XD3_ENCODER - CMD_ENCODE, -#endif - CMD_DECODE, - CMD_TEST, - CMD_CONFIG, -} xd3_cmd; - -#if XD3_ENCODER -#define CMD_DEFAULT CMD_ENCODE -#define IS_ENCODE(cmd) (cmd == CMD_ENCODE) -#else -#define CMD_DEFAULT CMD_DECODE -#define IS_ENCODE(cmd) (0) -#endif - -typedef struct _main_file main_file; -typedef struct _main_extcomp main_extcomp; -typedef struct _main_blklru main_blklru; -typedef struct _main_blklru_list main_blklru_list; -typedef struct _main_merge main_merge; -typedef struct _main_merge_list main_merge_list; - -/* The main_file object supports abstract system calls like open, - * close, read, write, seek, stat. The program uses these to - * represent both seekable files and non-seekable files. Source files - * must be seekable, but the target input and any output file do not - * require seekability. - */ -struct _main_file -{ -#if XD3_STDIO - FILE *file; -#elif XD3_POSIX - int file; -#elif XD3_WIN32 - HANDLE file; -#endif - - int mode; /* XO_READ and XO_WRITE */ - const char *filename; /* File name or /dev/stdin, - * /dev/stdout, /dev/stderr. */ - char *filename_copy; /* File name or /dev/stdin, - * /dev/stdout, /dev/stderr. */ - const char *realname; /* File name or /dev/stdin, - * /dev/stdout, /dev/stderr. */ - const main_extcomp *compressor; /* External compression struct. */ - int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ - xoff_t nread; /* for input position */ - xoff_t nwrite; /* for output position */ - uint8_t *snprintf_buf; /* internal snprintf() use */ -}; - -/* Various strings and magic values used to detect and call external - * compression. See below for examples. */ -struct _main_extcomp -{ - const char *recomp_cmdname; - const char *recomp_options; - - const char *decomp_cmdname; - const char *decomp_options; - - const char *ident; - const char *magic; - usize_t magic_size; - int flags; -}; - -/* This file implements a small LRU of source blocks. For encoding purposes, - * we prevent paging in blocks we've already scanned in the source (return - * XD3_NOTAVAIL). */ -struct _main_blklru_list -{ - main_blklru_list *next; - main_blklru_list *prev; -}; - -struct _main_blklru -{ - uint8_t *blk; - xoff_t blkno; - main_blklru_list link; -}; - -#define LRU_SIZE 32U -#define XD3_MINSRCWINSZ XD3_ALLOCSIZE - -/* ... represented as a list (no cache index). */ -XD3_MAKELIST(main_blklru_list,main_blklru,link); - -/* Merge state: */ - -struct _main_merge_list -{ - main_merge_list *next; - main_merge_list *prev; -}; - -struct _main_merge -{ - const char *filename; - - main_merge_list link; -}; - -XD3_MAKELIST(main_merge_list,main_merge,link); - -// TODO: really need to put options in a struct so that internal -// callers can easily reset state. - -/* Program options: various command line flags and options. */ -static int option_stdout = 0; -static int option_force = 0; -static int option_verbose = 0; -static int option_quiet = 0; -static int option_use_appheader = 1; -static uint8_t* option_appheader = NULL; -static int option_use_secondary = 0; -static char* option_secondary = NULL; -static int option_use_checksum = 1; -static int option_use_altcodetable = 0; -static char* option_smatch_config = NULL; -static int option_no_compress = 0; -static int option_no_output = 0; /* do not write output */ -static const char *option_source_filename = NULL; - -static int option_level = XD3_DEFAULT_LEVEL; -static usize_t option_iopt_size = XD3_DEFAULT_IOPT_SIZE; -static usize_t option_winsize = XD3_DEFAULT_WINSIZE; -static usize_t option_srcwinsz = XD3_DEFAULT_SRCWINSZ; -static usize_t option_sprevsz = XD3_DEFAULT_SPREVSZ; - -/* These variables are supressed to avoid their use w/o support. main() warns - * appropriately. */ -#if EXTERNAL_COMPRESSION -static int option_decompress_inputs = 1; -static int option_recompress_outputs = 1; -#endif - -/* This is for comparing "printdelta" output without attention to - * copy-instruction modes. */ -#if VCDIFF_TOOLS -static int option_print_cpymode = 1; /* Note: see reset_defaults(). */ -#endif - -/* Static variables */ -IF_DEBUG(static int main_mallocs = 0;) - -static char* program_name = NULL; -static uint8_t* appheader_used = NULL; -static uint8_t* main_bdata = NULL; -static usize_t main_bsize = 0; - -/* The LRU: obviously this is shared by all callers. */ -static usize_t lru_size = 0; -static main_blklru *lru = NULL; /* array of lru_size elts */ -static main_blklru_list lru_list; -static main_blklru_list lru_free; -static int do_not_lru = 0; /* set to avoid lru */ - -static int lru_hits = 0; -static int lru_misses = 0; -static int lru_filled = 0; - -/* Hacks for VCDIFF tools */ -static int allow_fake_source = 0; - -/* recode_stream is used by both recode/merge for reading vcdiff inputs */ -static xd3_stream *recode_stream = NULL; - -/* merge_stream is used by merge commands for storing the source encoding */ -static xd3_stream *merge_stream = NULL; - -/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is - * false just so the program knows the mapping of IDENT->NAME. */ -static main_extcomp extcomp_types[] = -{ - /* The entry for xdelta3 must be 0 because the program_name is set there. */ - { "xdelta3", "-cfq", "xdelta3", "-dcfq", "X", "\xd6\xc3\xc4", 3, - RD_NONEXTERNAL }, - { "bzip2", "-cf", "bzip2", "-dcf", "B", "BZh", 3, 0 }, - { "gzip", "-cf", "gzip", "-dcf", "G", "\037\213", 2, 0 }, - { "compress", "-cf", "uncompress", "-cf", "Z", "\037\235", 2, 0 }, - - /* TODO: add commandline support for magic-less formats */ - /*{ "lzma", "-cf", "lzma", "-dcf", "M", "]\000", 2, 0 },*/ -}; - -// }; - -static int main_input (xd3_cmd cmd, main_file *ifile, - main_file *ofile, main_file *sfile); -static void main_get_appheader (xd3_stream *stream, main_file *ifile, - main_file *output, main_file *sfile); - -static int main_help (void); - -static int -main_version (void) -{ - /* $Format: " DP(RINT \"Xdelta version $Xdelta3Version$, Copyright (C) 2007, 2008, Joshua MacDonald\n\");" $ */ - DP(RINT "Xdelta version 3.0u, Copyright (C) 2007, 2008, Joshua MacDonald\n"); - DP(RINT "Xdelta comes with ABSOLUTELY NO WARRANTY.\n"); - DP(RINT "This is free software, and you are welcome to redistribute it\n"); - DP(RINT "under certain conditions; see \"COPYING\" for details.\n"); - return EXIT_SUCCESS; -} - -static int -main_config (void) -{ - main_version (); - - DP(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION); - DP(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES); - DP(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE); - DP(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST); - DP(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW); - DP(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK); - DP(RINT "UNALIGNED_OK=%d\n", UNALIGNED_OK); - DP(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS); - DP(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE); - DP(RINT "XD3_DEBUG=%d\n", XD3_DEBUG); - DP(RINT "XD3_ENCODER=%d\n", XD3_ENCODER); - DP(RINT "XD3_POSIX=%d\n", XD3_POSIX); - DP(RINT "XD3_STDIO=%d\n", XD3_STDIO); - DP(RINT "XD3_WIN32=%d\n", XD3_WIN32); - DP(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64); - DP(RINT "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL); - DP(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE); - DP(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ); - DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ); - DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE); - DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE); - DP(RINT "sizeof(void*)=%ld\n", sizeof(void*)); - DP(RINT "sizeof(int)=%ld\n", sizeof(int)); - DP(RINT "sizeof(uint32_t)=%ld\n", sizeof(uint32_t)); - DP(RINT "sizeof(uint64_t)=%ld\n", sizeof(uint64_t)); - DP(RINT "sizeof(usize_t)=%ld\n", sizeof(usize_t)); - DP(RINT "sizeof(xoff_t)=%ld\n", sizeof(xoff_t)); - - return EXIT_SUCCESS; -} - -static void -reset_defaults(void) -{ - option_stdout = 0; - option_force = 0; - option_verbose = 0; - option_quiet = 0; - option_appheader = NULL; - option_use_secondary = 0; - option_secondary = NULL; - option_use_altcodetable = 0; - option_smatch_config = NULL; - option_no_compress = 0; - option_no_output = 0; - option_source_filename = NULL; - program_name = NULL; - appheader_used = NULL; - main_bdata = NULL; - main_bsize = 0; - lru_size = 0; - lru = NULL; - do_not_lru = 0; - lru_hits = 0; - lru_misses = 0; - lru_filled = 0; - allow_fake_source = 0; - option_smatch_config = NULL; - - option_use_appheader = 1; - option_use_checksum = 1; -#if EXTERNAL_COMPRESSION - option_decompress_inputs = 1; - option_recompress_outputs = 1; -#endif -#if VCDIFF_TOOLS - option_print_cpymode = 1; -#endif - option_level = XD3_DEFAULT_LEVEL; - option_iopt_size = XD3_DEFAULT_IOPT_SIZE; - option_winsize = XD3_DEFAULT_WINSIZE; - option_srcwinsz = XD3_DEFAULT_SRCWINSZ; - option_sprevsz = XD3_DEFAULT_SPREVSZ; -} - -static void* -main_malloc1 (usize_t size) -{ - void* r = malloc (size); - if (r == NULL) { XPR(NT "malloc: %s\n", xd3_mainerror (ENOMEM)); } - else if (option_verbose > 3) { XPR(NT "malloc: %u: %p\n", size, r); } - return r; -} - -static void* -main_malloc (usize_t size) -{ - void *r = main_malloc1 (size); - if (r) { IF_DEBUG (main_mallocs += 1); } - return r; -} - -static void* -main_alloc (void *opaque, - usize_t items, - usize_t size) -{ - return main_malloc1 (items * size); -} - -static void -main_free1 (void *opaque, void *ptr) -{ - if (option_verbose > 3) { XPR(NT "free: %p\n", ptr); } - free (ptr); -} - -static void -main_free (void *ptr) -{ - if (ptr) - { - IF_DEBUG (main_mallocs -= 1); - main_free1 (NULL, ptr); - IF_DEBUG (XD3_ASSERT(main_mallocs >= 0)); - } -} - -/* This ensures that (ret = errno) always indicates failure, in case errno was - * accidentally not set. If this prints there's a bug somewhere. */ -static int -get_errno (void) -{ -#ifndef _WIN32 - if (errno == 0) - { - XPR(NT "you found a bug: expected errno != 0\n"); - errno = XD3_INTERNAL; - } - return errno; -#else - DWORD errNum = GetLastError(); - if (errNum == NO_ERROR) { - errNum = XD3_INTERNAL; - } - return errNum; -#endif -} - -const char* -xd3_mainerror(int err_num) { -#ifndef _WIN32 - const char* x = xd3_strerror (err_num); - if (x != NULL) { - return x; - } - return strerror(err_num); -#else - static char err_buf[256]; - const char* x = xd3_strerror (err_num); - if (x != NULL) { - return x; - } - memset (err_buf, 0, 256); - FormatMessage (FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, err_num, - MAKELANGID (LANG_NEUTRAL, SUBLANG_DEFAULT), - err_buf, 256, NULL); - return err_buf; -#endif -} - -static long -get_millisecs_now (void) -{ -#ifndef _WIN32 - struct timeval tv; - - gettimeofday (& tv, NULL); - - return (tv.tv_sec) * 1000L + (tv.tv_usec) / 1000; -#else - SYSTEMTIME st; - FILETIME ft; - __int64 *pi = (__int64*)&ft; - GetLocalTime(&st); - SystemTimeToFileTime(&st, &ft); - return (long)((*pi) / 10000); -#endif -} - -/* Always >= 1 millisec, right? */ -static long -get_millisecs_since (void) -{ - static long last = 0; - long now = get_millisecs_now(); - long diff = now - last; - last = now; - return diff; -} - -static char* -main_format_bcnt (xoff_t r, char *buf) -{ - static const char* fmts[] = { "B", "KB", "MB", "GB" }; - usize_t i; - - for (i = 0; i < SIZEOF_ARRAY(fmts); i += 1) - { - if (r <= (10 * 1024) || i == (-1 + (int)SIZEOF_ARRAY(fmts))) - { - sprintf (buf, "%"Q"u %s", r, fmts[i]); - break; - } - r /= 1024; - } - return buf; -} - -static char* -main_format_rate (xoff_t bytes, long millis, char *buf) -{ - xoff_t r = (xoff_t)(1.0 * bytes / (1.0 * millis / 1000.0)); - static char lbuf[32]; - - main_format_bcnt (r, lbuf); - sprintf (buf, "%s/sec", lbuf); - return buf; -} - -static char* -main_format_millis (long millis, char *buf) -{ - if (millis < 1000) { sprintf (buf, "%lu ms", millis); } - else if (millis < 10000) { sprintf (buf, "%.1f sec", millis / 1000.0); } - else { sprintf (buf, "%lu sec", millis / 1000L); } - return buf; -} - -/* A safe version of strtol for xoff_t. */ -static int -main_strtoxoff (const char* s, xoff_t *xo, char which) -{ - char *e; - xoff_t x; - - XD3_ASSERT(s && *s != 0); - - { - /* Should check LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX? */ -#if SIZEOF_XOFF_T == 4 - long xx = strtol (s, &e, 0); -#else - long long xx = strtoll (s, &e, 0); -#endif - - if (xx < 0) - { - XPR(NT "-%c: negative integer: %s\n", which, s); - return EXIT_FAILURE; - } - - x = xx; - } - - if (*e != 0) - { - XPR(NT "-%c: invalid integer: %s\n", which, s); - return EXIT_FAILURE; - } - - (*xo) = x; - return 0; -} - -static int -main_atou (const char* arg, usize_t *xo, usize_t low, - usize_t high, char which) -{ - xoff_t x; - int ret; - - if ((ret = main_strtoxoff (arg, & x, which))) { return ret; } - - if (x < low) - { - XPR(NT "-%c: minimum value: %u\n", which, low); - return EXIT_FAILURE; - } - if (high == 0) - { - high = USIZE_T_MAX; - } - if (x > high) - { - XPR(NT "-%c: maximum value: %u\n", which, high); - return EXIT_FAILURE; - } - (*xo) = (usize_t)x; - return 0; -} - -/****************************************************************** - FILE BASICS - ******************************************************************/ - -/* With all the variation in file system-call semantics, arguments, - * return values and error-handling for the POSIX and STDIO file APIs, - * the insides of these functions make me sick, which is why these - * wrappers exist. */ - -#define XOPEN_OPNAME (xfile->mode == XO_READ ? "read" : "write") -#define XOPEN_STDIO (xfile->mode == XO_READ ? "rb" : "wb") -#define XOPEN_POSIX (xfile->mode == XO_READ ? \ - O_RDONLY : O_WRONLY | O_CREAT | O_TRUNC) -#define XOPEN_MODE (xfile->mode == XO_READ ? 0 : 0666) - -#define XF_ERROR(op, name, ret) \ - do { if (!option_quiet) { XPR(NT "file %s failed: %s: %s: %s\n", (op), \ - XOPEN_OPNAME, (name), xd3_mainerror (ret)); } } while (0) - -#if XD3_STDIO -#define XFNO(f) fileno(f->file) -#define XSTDOUT_XF(f) { (f)->file = stdout; (f)->filename = "/dev/stdout"; } -#define XSTDIN_XF(f) { (f)->file = stdin; (f)->filename = "/dev/stdin"; } - -#elif XD3_POSIX -#define XFNO(f) f->file -#define XSTDOUT_XF(f) \ - { (f)->file = STDOUT_FILENO; (f)->filename = "/dev/stdout"; } -#define XSTDIN_XF(f) \ - { (f)->file = STDIN_FILENO; (f)->filename = "/dev/stdin"; } - -#elif XD3_WIN32 -#define XFNO(f) -1 -#define XSTDOUT_XF(f) { \ - (f)->file = GetStdHandle(STD_OUTPUT_HANDLE); \ - (f)->filename = "(stdout)"; \ - } -#define XSTDIN_XF(f) { \ - (f)->file = GetStdHandle(STD_INPUT_HANDLE); \ - (f)->filename = "(stdin)"; \ - } -#endif - -static void -main_file_init (main_file *xfile) -{ - memset (xfile, 0, sizeof (*xfile)); - -#if XD3_POSIX - xfile->file = -1; -#endif -#if XD3_WIN32 - xfile->file = INVALID_HANDLE_VALUE; -#endif -} - -static int -main_file_isopen (main_file *xfile) -{ -#if XD3_STDIO - return xfile->file != NULL; - -#elif XD3_POSIX - return xfile->file != -1; - -#elif XD3_WIN32 - return xfile->file != INVALID_HANDLE_VALUE; -#endif -} - -static int -main_file_close (main_file *xfile) -{ - int ret = 0; - - if (! main_file_isopen (xfile)) - { - return 0; - } - -#if XD3_STDIO - ret = fclose (xfile->file); - xfile->file = NULL; - -#elif XD3_POSIX - ret = close (xfile->file); - xfile->file = -1; - -#elif XD3_WIN32 - if (!CloseHandle(xfile->file)) { - ret = get_errno (); - } - xfile->file = INVALID_HANDLE_VALUE; -#endif - - if (ret != 0) { XF_ERROR ("close", xfile->filename, ret = get_errno ()); } - return ret; -} - -static void -main_file_cleanup (main_file *xfile) -{ - XD3_ASSERT (xfile != NULL); - - if (main_file_isopen (xfile)) - { - main_file_close (xfile); - } - - if (xfile->snprintf_buf != NULL) - { - main_free(xfile->snprintf_buf); - xfile->snprintf_buf = NULL; - } - - if (xfile->filename_copy != NULL) - { - main_free(xfile->filename_copy); - xfile->filename_copy = NULL; - } -} - -static int -main_file_open (main_file *xfile, const char* name, int mode) -{ - int ret = 0; - - xfile->mode = mode; - - XD3_ASSERT (name != NULL); - XD3_ASSERT (! main_file_isopen (xfile)); - if (name[0] == 0) - { - XPR(NT "invalid file name: empty string\n"); - return XD3_INVALID; - } - -#if XD3_STDIO - xfile->file = fopen (name, XOPEN_STDIO); - - ret = (xfile->file == NULL) ? get_errno () : 0; - -#elif XD3_POSIX - if ((ret = open (name, XOPEN_POSIX, XOPEN_MODE)) < 0) - { - ret = get_errno (); - } - else - { - xfile->file = ret; - ret = 0; - } - -#elif XD3_WIN32 - xfile->file = CreateFile(name, - (mode == XO_READ) ? GENERIC_READ : GENERIC_WRITE, - FILE_SHARE_READ, - NULL, - (mode == XO_READ) ? OPEN_EXISTING : - (option_force ? CREATE_ALWAYS : CREATE_NEW), - FILE_ATTRIBUTE_NORMAL, - NULL); - if (xfile->file == INVALID_HANDLE_VALUE) { - ret = get_errno (); - } -#endif - if (ret) { XF_ERROR ("open", name, ret); } - else { xfile->realname = name; xfile->nread = 0; } - return ret; -} - -static int -main_file_stat (main_file *xfile, xoff_t *size, int err_ifnoseek) -{ - int ret = 0; -#if XD3_WIN32 -# if (_WIN32_WINNT >= 0x0500) - LARGE_INTEGER li; - if (GetFileSizeEx(xfile->file, &li) == 0) - { - ret = get_errno (); - } - else - { - *size = li.QuadPart; - } -# else - DWORD filesize = GetFileSize(xfile->file, NULL); - if (filesize == INVALID_FILE_SIZE) - { - ret = GetLastError(); - if (ret != NO_ERROR) - return ret; - } - *size = filesize; -# endif -#else - struct stat sbuf; - if (fstat (XFNO (xfile), & sbuf) < 0) - { - ret = get_errno (); - if (err_ifnoseek) - { - XF_ERROR ("stat", xfile->filename, ret); - } - return ret; - } - - if (! S_ISREG (sbuf.st_mode)) - { - if (err_ifnoseek) - { - XPR(NT "source file must be seekable: %s\n", xfile->filename); - } - return ESPIPE; - } - (*size) = sbuf.st_size; -#endif - return ret; -} - -static int -main_file_exists (main_file *xfile) -{ - struct stat sbuf; - return stat (xfile->filename, & sbuf) == 0 && S_ISREG (sbuf.st_mode); -} - -#if (XD3_POSIX || EXTERNAL_COMPRESSION) -/* POSIX-generic code takes a function pointer to read() or write(). - * This calls the function repeatedly until the buffer is full or EOF. - * The NREAD parameter is not set for write, NULL is passed. Return - * is signed, < 0 indicate errors, otherwise byte count. */ -typedef int (xd3_posix_func) (int fd, uint8_t *buf, usize_t size); - -static int -xd3_posix_io (int fd, uint8_t *buf, usize_t size, - xd3_posix_func *func, usize_t *nread) -{ - int ret; - usize_t nproc = 0; - - while (nproc < size) - { - int result = (*func) (fd, buf + nproc, size - nproc); - - if (result < 0) - { - ret = get_errno (); - if (ret != EAGAIN && ret != EINTR) - { - return ret; - } - result = 0; - } - - if (nread != NULL && result == 0) { break; } - - nproc += result; - } - if (nread != NULL) { (*nread) = nproc; } - return 0; -} -#endif - -/* POSIX is unbuffered, while STDIO is buffered. main_file_read() - * should always be called on blocks. */ -static int -main_file_read (main_file *ifile, - uint8_t *buf, - usize_t size, - usize_t *nread, - const char *msg) -{ - int ret = 0; - -#if XD3_STDIO - usize_t result; - - result = fread (buf, 1, size, ifile->file); - - if (result < size && ferror (ifile->file)) - { - ret = get_errno (); - } - else - { - *nread = result; - } - -#elif XD3_POSIX - ret = xd3_posix_io (ifile->file, buf, size, (xd3_posix_func*) &read, nread); - -#elif XD3_WIN32 - DWORD nread2; - if (ReadFile (ifile->file, buf, size, &nread2, NULL) == 0) { - ret = get_errno(); - } else { - *nread = (usize_t)nread2; - } -#endif - - if (ret) - { - XPR(NT "%s: %s: %s\n", msg, ifile->filename, xd3_mainerror (ret)); - } - else - { - if (option_verbose > 3) { XPR(NT "main read: %s: %u\n", - ifile->filename, (*nread)); } - ifile->nread += (*nread); - } - - return ret; -} - -static int -main_file_write (main_file *ofile, uint8_t *buf, usize_t size, const char *msg) -{ - int ret = 0; - -#if XD3_STDIO - usize_t result; - - result = fwrite (buf, 1, size, ofile->file); - - if (result != size) { ret = get_errno (); } - -#elif XD3_POSIX - ret = xd3_posix_io (ofile->file, buf, size, (xd3_posix_func*) &write, NULL); - -#elif XD3_WIN32 - DWORD nwrite; - if (WriteFile(ofile->file, buf, size, &nwrite, NULL) == 0) { - ret = get_errno (); - } else { - if (size != nwrite) { - XPR(NT "Incorrect write count"); - ret = XD3_INTERNAL; - } - } -#endif - - if (ret) - { - XPR(NT "%s: %s: %s\n", msg, ofile->filename, xd3_mainerror (ret)); - } - else - { - if (option_verbose > 3) { XPR(NT "main write: %s: %u\n", - ofile->filename, size); } - ofile->nwrite += size; - } - - return ret; -} - -static int -main_file_seek (main_file *xfile, xoff_t pos) -{ - int ret = 0; - -#if XD3_STDIO - if (fseek (xfile->file, pos, SEEK_SET) != 0) { ret = get_errno (); } - -#elif XD3_POSIX - if ((xoff_t) lseek (xfile->file, pos, SEEK_SET) != pos) - { ret = get_errno (); } - -#elif XD3_WIN32 -# if (_WIN32_WINNT >= 0x0500) - LARGE_INTEGER move, out; - move.QuadPart = pos; - if (SetFilePointerEx(xfile->file, move, &out, FILE_BEGIN) == 0) { - ret = get_errno (); - } -# else - if (SetFilePointer(xfile->file, (LONG)pos, NULL, FILE_BEGIN) == - INVALID_SET_FILE_POINTER) - { - ret = get_errno (); - } -# endif -#endif - - if (ret) - { - XPR(NT "seek failed: %s: %s\n", xfile->filename, xd3_mainerror (ret)); - } - - return ret; -} - -/* This function simply writes the stream output buffer, if there is - * any, for encode, decode and recode commands. (The VCDIFF tools use - * main_print_func()). */ -static int -main_write_output (xd3_stream* stream, main_file *ofile) -{ - int ret; - - if (option_no_output) - { - return 0; - } - - if (stream->avail_out > 0 && - (ret = main_file_write (ofile, stream->next_out, - stream->avail_out, "write failed"))) - { - return ret; - } - - return 0; -} - -static int -main_set_secondary_flags (xd3_config *config) -{ - int ret; - if (option_use_secondary) - { - /* The default secondary compressor is DJW, if it's compiled. */ - if (option_secondary == NULL) - { - if (SECONDARY_DJW) - { - config->flags |= XD3_SEC_DJW; - } - } - else - { - if (strcmp (option_secondary, "fgk") == 0 && SECONDARY_FGK) - { - config->flags |= XD3_SEC_FGK; - } - else if (strncmp (option_secondary, "djw", 3) == 0 && SECONDARY_DJW) - { - usize_t level = XD3_DEFAULT_SECONDARY_LEVEL; - - config->flags |= XD3_SEC_DJW; - - if (strlen (option_secondary) > 3 && - (ret = main_atou (option_secondary + 3, - &level, - 0, 9, 'S')) != 0 && - !option_quiet) - { - return XD3_INVALID; - } - - /* XD3_SEC_NOXXXX flags disable secondary compression on - * a per-section basis. For djw, ngroups=1 indicates - * minimum work, ngroups=0 uses default settings, which - * is > 1 groups by default. */ - if (level < 1) { config->flags |= XD3_SEC_NODATA; } - if (level < 7) { config->sec_data.ngroups = 1; } - else { config->sec_data.ngroups = 0; } - - if (level < 3) { config->flags |= XD3_SEC_NOINST; } - if (level < 8) { config->sec_inst.ngroups = 1; } - else { config->sec_inst.ngroups = 0; } - - if (level < 5) { config->flags |= XD3_SEC_NOADDR; } - if (level < 9) { config->sec_addr.ngroups = 1; } - else { config->sec_addr.ngroups = 0; } - } - else if (strcmp (option_secondary, "none") == 0 && SECONDARY_DJW) - { - /* No secondary */ - } - else - { - if (!option_quiet) - { - XPR(NT "unrecognized secondary compressor type: %s\n", - option_secondary); - return XD3_INVALID; - } - } - } - } - - return 0; -} - -/****************************************************************** - VCDIFF TOOLS - *****************************************************************/ - -#if VCDIFF_TOOLS -#include "xdelta3-merge.h" - -#if defined(_WIN32) || defined(__DJGPP__) -/* According to the internet, Windows vsnprintf() differs from most - * Unix implementations regarding the terminating 0 when the boundary - * condition is met. It doesn't matter here, we don't rely on the - * trailing 0. Besides, both Windows and DJGPP vsnprintf return -1 - * upon truncation, which isn't C99 compliant. To overcome this, - * recent MinGW runtimes provided their own vsnprintf (notice the - * absence of the '_' prefix) but they were initially buggy. So, - * always use the native '_'-prefixed version with Win32. */ -#include <stdarg.h> -#ifdef _WIN32 -#define vsnprintf_func _vsnprintf -#else -#define vsnprintf_func vsnprintf -#endif - -int -snprintf_func (char *str, int n, char *fmt, ...) -{ - va_list a; - int ret; - va_start (a, fmt); - ret = vsnprintf_func (str, n, fmt, a); - va_end (a); - if (ret < 0) - ret = n; - return ret; -} -#else -#define snprintf_func snprintf -#endif - -/* The following macros let VCDIFF printing something printf-like with - * main_file_write(), e.g.,: - * - * VC(UT "trying to be portable: %d\n", x)VE; - */ -#define SNPRINTF_BUFSIZE 1024 -#define VC do { if (((ret = snprintf_func -#define UT (char*)xfile->snprintf_buf, SNPRINTF_BUFSIZE, -#define VE ) >= SNPRINTF_BUFSIZE \ - && (ret = main_print_overflow(ret)) != 0) \ - || (ret = main_file_write(xfile, xfile->snprintf_buf, \ - ret, "print")) != 0) \ - { return ret; } } while (0) - -static int -main_print_overflow (int x) -{ - XPR(NT "internal print buffer overflow: %d bytes\n", x); - return XD3_INTERNAL; -} - -/* This function prints a single VCDIFF window. */ -static int -main_print_window (xd3_stream* stream, main_file *xfile) -{ - int ret; - usize_t size = 0; - - VC(UT " Offset Code Type1 Size1 @Addr1 + Type2 Size2 @Addr2\n")VE; - - while (stream->inst_sect.buf < stream->inst_sect.buf_max) - { - usize_t code = stream->inst_sect.buf[0]; - const uint8_t *addr_before = stream->addr_sect.buf; - const uint8_t *inst_before = stream->inst_sect.buf; - usize_t addr_bytes; - usize_t inst_bytes; - usize_t size_before = size; - - if ((ret = xd3_decode_instruction (stream))) - { - XPR(NT "instruction decode error at %"Q"u: %s\n", - stream->dec_winstart + size, stream->msg); - return ret; - } - - addr_bytes = stream->addr_sect.buf - addr_before; - inst_bytes = stream->inst_sect.buf - inst_before; - - VC(UT " %06"Q"u %03u %s %6u", stream->dec_winstart + size, - option_print_cpymode ? code : 0, - xd3_rtype_to_string ((xd3_rtype) stream->dec_current1.type, option_print_cpymode), - (usize_t) stream->dec_current1.size)VE; - - if (stream->dec_current1.type != XD3_NOOP) - { - if (stream->dec_current1.type >= XD3_CPY) - { - if (stream->dec_current1.addr >= stream->dec_cpylen) - { - VC(UT " T@%-6u", - stream->dec_current1.addr - stream->dec_cpylen)VE; - } - else - { - VC(UT " S@%-6"Q"u", - stream->dec_cpyoff + stream->dec_current1.addr)VE; - } - } - else - { - VC(UT " ")VE; - } - - size += stream->dec_current1.size; - } - - if (stream->dec_current2.type != XD3_NOOP) - { - VC(UT " %s %6u", - xd3_rtype_to_string ((xd3_rtype) stream->dec_current2.type, - option_print_cpymode), - (usize_t)stream->dec_current2.size)VE; - - if (stream->dec_current2.type >= XD3_CPY) - { - if (stream->dec_current2.addr >= stream->dec_cpylen) - { - VC(UT " T@%-6u", - stream->dec_current2.addr - stream->dec_cpylen)VE; - } - else - { - VC(UT " S@%-6"Q"u", - stream->dec_cpyoff + stream->dec_current2.addr)VE; - } - } - - size += stream->dec_current2.size; - } - - VC(UT "\n")VE; - - if (option_verbose && - addr_bytes + inst_bytes >= (size - size_before) && - (stream->dec_current1.type >= XD3_CPY || - stream->dec_current2.type >= XD3_CPY)) - { - VC(UT " %06"Q"u (inefficiency) %u encoded as %u bytes\n", - stream->dec_winstart + size_before, - size - size_before, - addr_bytes + inst_bytes)VE; - } - } - - if (stream->dec_tgtlen != size && (stream->flags & XD3_SKIP_WINDOW) == 0) - { - XPR(NT "target window size inconsistency"); - return XD3_INTERNAL; - } - - if (stream->dec_position != stream->dec_maxpos) - { - XPR(NT "target window position inconsistency"); - return XD3_INTERNAL; - } - - if (stream->addr_sect.buf != stream->addr_sect.buf_max) - { - XPR(NT "address section inconsistency"); - return XD3_INTERNAL; - } - - return 0; -} - -static int -main_print_vcdiff_file (main_file *xfile, main_file *file, const char *type) -{ - int ret; /* Used by above macros */ - if (file->filename) - { - VC(UT "XDELTA filename (%s): %s\n", type, - file->filename)VE; - } - if (file->compressor) - { - VC(UT "XDELTA ext comp (%s): %s\n", type, - file->compressor->recomp_cmdname)VE; - } - return 0; -} - -/* This function prints a VCDIFF input, mainly for debugging purposes. */ -static int -main_print_func (xd3_stream* stream, main_file *xfile) -{ - int ret; - - if (option_no_output) - { - return 0; - } - - if (xfile->snprintf_buf == NULL) - { - if ((xfile->snprintf_buf = (uint8_t*)main_malloc(SNPRINTF_BUFSIZE)) == NULL) - { - return ENOMEM; - } - } - - if (stream->dec_winstart == 0) - { - VC(UT "VCDIFF version: 0\n")VE; - VC(UT "VCDIFF header size: %d\n", - stream->dec_hdrsize)VE; - VC(UT "VCDIFF header indicator: ")VE; - if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) - VC(UT "VCD_SECONDARY ")VE; - if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) - VC(UT "VCD_CODETABLE ")VE; - if ((stream->dec_hdr_ind & VCD_APPHEADER) != 0) - VC(UT "VCD_APPHEADER ")VE; - if (stream->dec_hdr_ind == 0) - VC(UT "none")VE; - VC(UT "\n")VE; - - IF_SEC(VC(UT "VCDIFF secondary compressor: %s\n", - stream->sec_type ? stream->sec_type->name : "none")VE); - IF_NSEC(VC(UT "VCDIFF secondary compressor: unsupported\n")VE); - - if (stream->dec_hdr_ind & VCD_APPHEADER) - { - uint8_t *apphead; - usize_t appheadsz; - ret = xd3_get_appheader (stream, & apphead, & appheadsz); - - if (ret == 0 && appheadsz > 0) - { - int sq = option_quiet; - main_file i, o, s; - XD3_ASSERT (apphead != NULL); - VC(UT "VCDIFF application header: ")VE; - if ((ret = main_file_write (xfile, apphead, - appheadsz, "print")) != 0) - { return ret; } - VC(UT "\n")VE; - - main_file_init (& i); - main_file_init (& o); - main_file_init (& s); - option_quiet = 1; - main_get_appheader (stream, &i, & o, & s); - option_quiet = sq; - if ((ret = main_print_vcdiff_file (xfile, & o, "output"))) - { return ret; } - if ((ret = main_print_vcdiff_file (xfile, & s, "source"))) - { return ret; } - main_file_cleanup (& i); - main_file_cleanup (& o); - main_file_cleanup (& s); - } - } - } - else - { - VC(UT "\n")VE; - } - - VC(UT "VCDIFF window number: %"Q"u\n", stream->current_window)VE; - VC(UT "VCDIFF window indicator: ")VE; - if ((stream->dec_win_ind & VCD_SOURCE) != 0) VC(UT "VCD_SOURCE ")VE; - if ((stream->dec_win_ind & VCD_TARGET) != 0) VC(UT "VCD_TARGET ")VE; - if ((stream->dec_win_ind & VCD_ADLER32) != 0) VC(UT "VCD_ADLER32 ")VE; - if (stream->dec_win_ind == 0) VC(UT "none")VE; - VC(UT "\n")VE; - - if ((stream->dec_win_ind & VCD_ADLER32) != 0) - { - VC(UT "VCDIFF adler32 checksum: %08X\n", - (usize_t)stream->dec_adler32)VE; - } - - if (stream->dec_del_ind != 0) - { - VC(UT "VCDIFF delta indicator: ")VE; - if ((stream->dec_del_ind & VCD_DATACOMP) != 0) VC(UT "VCD_DATACOMP ")VE; - if ((stream->dec_del_ind & VCD_INSTCOMP) != 0) VC(UT "VCD_INSTCOMP ")VE; - if ((stream->dec_del_ind & VCD_ADDRCOMP) != 0) VC(UT "VCD_ADDRCOMP ")VE; - if (stream->dec_del_ind == 0) VC(UT "none")VE; - VC(UT "\n")VE; - } - - if (stream->dec_winstart != 0) - { - VC(UT "VCDIFF window at offset: %"Q"u\n", stream->dec_winstart)VE; - } - - if (SRCORTGT (stream->dec_win_ind)) - { - VC(UT "VCDIFF copy window length: %u\n", - (usize_t)stream->dec_cpylen)VE; - VC(UT "VCDIFF copy window offset: %"Q"u\n", - stream->dec_cpyoff)VE; - } - - VC(UT "VCDIFF delta encoding length: %u\n", - (usize_t)stream->dec_enclen)VE; - VC(UT "VCDIFF target window length: %u\n", - (usize_t)stream->dec_tgtlen)VE; - - VC(UT "VCDIFF data section length: %u\n", - (usize_t)stream->data_sect.size)VE; - VC(UT "VCDIFF inst section length: %u\n", - (usize_t)stream->inst_sect.size)VE; - VC(UT "VCDIFF addr section length: %u\n", - (usize_t)stream->addr_sect.size)VE; - - ret = 0; - if ((stream->flags & XD3_JUST_HDR) != 0) - { - /* Print a header -- finished! */ - ret = PRINTHDR_SPECIAL; - } - else if ((stream->flags & XD3_SKIP_WINDOW) == 0) - { - ret = main_print_window (stream, xfile); - } - - return ret; -} - -static int -main_recode_copy (xd3_stream* stream, - xd3_output* output, - xd3_desect* input) -{ - int ret; - - XD3_ASSERT(output != NULL); - XD3_ASSERT(output->next_page == NULL); - - if ((ret = xd3_decode_allocate (recode_stream, - input->size, - &output->base, - &output->avail))) - { - XPR(NT XD3_LIB_ERRMSG (stream, ret)); - return ret; - } - - memcpy (output->base, - /* Note: decoder advances buf, so get base of buffer with - * buf_max - size */ - input->buf_max - input->size, - input->size); - output->next = input->size; - return 0; -} - -// Re-encode one window -static int -main_recode_func (xd3_stream* stream, main_file *ofile) -{ - int ret; - xd3_source decode_source; - - XD3_ASSERT(stream->dec_state == DEC_FINISH); - XD3_ASSERT(recode_stream->enc_state == ENC_INIT || - recode_stream->enc_state == ENC_INPUT); - - // Copy partial decoder output to partial encoder inputs - if ((ret = main_recode_copy (recode_stream, - DATA_HEAD(recode_stream), - &stream->data_sect)) || - (ret = main_recode_copy (recode_stream, - INST_HEAD(recode_stream), - &stream->inst_sect)) || - (ret = main_recode_copy (recode_stream, - ADDR_HEAD(recode_stream), - &stream->addr_sect))) - { - return ret; - } - - // This jumps to xd3_emit_hdr() - recode_stream->enc_state = ENC_FLUSH; - recode_stream->avail_in = stream->dec_tgtlen; - - if (SRCORTGT (stream->dec_win_ind)) - { - recode_stream->src = & decode_source; - decode_source.srclen = stream->dec_cpylen; - decode_source.srcbase = stream->dec_cpyoff; - } - - if (option_use_checksum && - (stream->dec_win_ind & VCD_ADLER32) != 0) - { - recode_stream->flags |= XD3_ADLER32_RECODE; - recode_stream->recode_adler32 = stream->dec_adler32; - } - - if (option_use_appheader != 0 && - option_appheader != NULL) - { - xd3_set_appheader (recode_stream, option_appheader, - strlen ((char*) option_appheader)); - } - else if (option_use_appheader != 0 && - option_appheader == NULL) - { - if (stream->dec_appheader != NULL) - { - xd3_set_appheader (recode_stream, - stream->dec_appheader, stream->dec_appheadsz); - } - } - - // Output loop - for (;;) - { - switch((ret = xd3_encode_input (recode_stream))) - { - case XD3_INPUT: { - /* finished recoding one window */ - stream->total_out = recode_stream->total_out; - return 0; - } - case XD3_OUTPUT: { - /* main_file_write below */ - break; - } - case XD3_GOTHEADER: - case XD3_WINSTART: - case XD3_WINFINISH: { - /* ignore */ - continue; - } - case XD3_GETSRCBLK: - case 0: { - return XD3_INTERNAL; - } - default: - return ret; - } - - if ((ret = main_write_output (recode_stream, ofile))) - { - return ret; - } - - xd3_consume_output (recode_stream); - } -} -#endif /* VCDIFF_TOOLS */ - -/******************************************************************* - VCDIFF merging - ******************************************************************/ - -#if VCDIFF_TOOLS -/* Modifies static state. */ -static int -main_init_recode_stream (void) -{ - int ret; - int stream_flags = XD3_ADLER32_NOVER | XD3_SKIP_EMIT; - int recode_flags; - xd3_config recode_config; - - XD3_ASSERT (recode_stream == NULL); - - if ((recode_stream = (xd3_stream*) main_malloc(sizeof(xd3_stream))) == NULL) - { - return ENOMEM; - } - - recode_flags = (stream_flags & XD3_SEC_TYPE); - - recode_config.alloc = main_alloc; - recode_config.freef = main_free1; - - xd3_init_config(&recode_config, recode_flags); - - if ((ret = main_set_secondary_flags (&recode_config)) || - (ret = xd3_config_stream (recode_stream, &recode_config)) || - (ret = xd3_encode_init_partial (recode_stream)) || - (ret = xd3_whole_state_init (recode_stream))) - { - XPR(NT XD3_LIB_ERRMSG (recode_stream, ret)); - xd3_free_stream (recode_stream); - recode_stream = NULL; - return ret; - } - - return 0; -} - -/* This processes the sequence of -m arguments. The final input - * is processed as part of the ordinary main_input() loop. */ -static int -main_merge_arguments (main_merge_list* merges) -{ - int ret = 0; - int count = 0; - main_merge *merge = NULL; - xd3_stream merge_input; - - if (main_merge_list_empty (merges)) - { - return 0; - } - - if ((ret = xd3_config_stream (& merge_input, NULL)) || - (ret = xd3_whole_state_init (& merge_input))) - { - XPR(NT XD3_LIB_ERRMSG (& merge_input, ret)); - return ret; - } - - merge = main_merge_list_front (merges); - while (!main_merge_list_end (merges, merge)) - { - main_file mfile; - main_file_init (& mfile); - mfile.filename = merge->filename; - mfile.flags = RD_NONEXTERNAL; - - if ((ret = main_file_open (& mfile, merge->filename, XO_READ))) - { - goto error; - } - - ret = main_input (CMD_MERGE_ARG, & mfile, NULL, NULL); - - if (ret == 0) - { - if (count++ == 0) - { - /* The first merge source is the next merge input. */ - xd3_swap_whole_state (& recode_stream->whole_target, - & merge_input.whole_target); - } - else - { - /* Merge the recode_stream with merge_input. */ - ret = xd3_merge_input_output (recode_stream, - & merge_input.whole_target); - - /* Save the next merge source in merge_input. */ - xd3_swap_whole_state (& recode_stream->whole_target, - & merge_input.whole_target); - } - } - - main_file_cleanup (& mfile); - - if (recode_stream != NULL) - { - xd3_free_stream (recode_stream); - main_free (recode_stream); - recode_stream = NULL; - } - - if (main_bdata != NULL) - { - main_free (main_bdata); - main_bdata = NULL; - main_bsize = 0; - } - - if (ret != 0) - { - goto error; - } - - merge = main_merge_list_next (merge); - } - - XD3_ASSERT (merge_stream == NULL); - - if ((merge_stream = (xd3_stream*) main_malloc (sizeof(xd3_stream))) == NULL) - { - ret = ENOMEM; - goto error; - } - - if ((ret = xd3_config_stream (merge_stream, NULL)) || - (ret = xd3_whole_state_init (merge_stream))) - { - XPR(NT XD3_LIB_ERRMSG (& merge_input, ret)); - goto error; - } - - xd3_swap_whole_state (& merge_stream->whole_target, - & merge_input.whole_target); - ret = 0; - error: - xd3_free_stream (& merge_input); - return ret; -} - -/* This processes each window of the final merge input. This routine - * does not output, it buffers the entire delta into memory. */ -static int -main_merge_func (xd3_stream* stream, main_file *no_write) -{ - int ret; - - if ((ret = xd3_whole_append_window (stream))) - { - return ret; - } - - return 0; -} - - -/* This is called after all windows have been read, as a final step in - * main_input(). This is only called for the final merge step. */ -static int -main_merge_output (xd3_stream *stream, main_file *ofile) -{ - int ret; - usize_t inst_pos = 0; - xoff_t output_pos = 0; - xd3_source recode_source; - usize_t window_num = 0; - int at_least_once = 0; - - /* merge_stream is set if there were arguments. this stream's input - * needs to be applied to the merge_stream source. */ - if ((merge_stream != NULL) && - (ret = xd3_merge_input_output (stream, - & merge_stream->whole_target))) - { - XPR(NT XD3_LIB_ERRMSG (stream, ret)); - return ret; - } - - if (option_use_appheader != 0 && - option_appheader != NULL) - { - xd3_set_appheader (recode_stream, option_appheader, - strlen ((char*) option_appheader)); - } - - /* Enter the ENC_INPUT state and bypass the next_in == NULL test - * and (leftover) input buffering logic. */ - XD3_ASSERT(recode_stream->enc_state == ENC_INIT); - recode_stream->enc_state = ENC_INPUT; - recode_stream->next_in = main_bdata; - recode_stream->flags |= XD3_FLUSH; - - /* This encodes the entire target. */ - while (inst_pos < stream->whole_target.instlen || !at_least_once) - { - xoff_t window_start = output_pos; - int window_srcset = 0; - xoff_t window_srcmin = 0; - xoff_t window_srcmax = 0; - usize_t window_pos = 0; - usize_t window_size; - - /* at_least_once ensures that we encode at least one window, - * which handles the 0-byte case. */ - at_least_once = 1; - - XD3_ASSERT (recode_stream->enc_state == ENC_INPUT); - - if ((ret = xd3_encode_input (recode_stream)) != XD3_WINSTART) - { - XPR(NT "invalid merge state: %s\n", xd3_mainerror (ret)); - return XD3_INVALID; - } - - /* Window sizes must match from the input to the output, so that - * target copies are in-range (and so that checksums carry - * over). */ - XD3_ASSERT (window_num < stream->whole_target.wininfolen); - window_size = stream->whole_target.wininfo[window_num].length; - - /* Output position should also match. */ - if (output_pos != stream->whole_target.wininfo[window_num].offset) - { - XPR(NT "internal merge error: offset mismatch\n"); - return XD3_INVALID; - } - - if (option_use_checksum && - (stream->dec_win_ind & VCD_ADLER32) != 0) - { - recode_stream->flags |= XD3_ADLER32_RECODE; - recode_stream->recode_adler32 = stream->whole_target.wininfo[window_num].adler32; - } - - window_num++; - - if (main_bsize < window_size) - { - main_free (main_bdata); - main_bdata = NULL; - main_bsize = 0; - if ((main_bdata = (uint8_t*) - main_malloc (window_size)) == NULL) - { - return ENOMEM; - } - main_bsize = window_size; - } - - /* This encodes a single target window. */ - while (window_pos < window_size && - inst_pos < stream->whole_target.instlen) - { - xd3_winst *inst = &stream->whole_target.inst[inst_pos]; - usize_t take = min(inst->size, window_size - window_pos); - xoff_t addr; - - switch (inst->type) - { - case XD3_RUN: - if ((ret = xd3_emit_run (recode_stream, window_pos, take, - stream->whole_target.adds[inst->addr]))) - { - return ret; - } - break; - - case XD3_ADD: - /* Adds are implicit, put them into the input buffer. */ - memcpy (main_bdata + window_pos, - stream->whole_target.adds + inst->addr, take); - break; - - default: /* XD3_COPY + copy mode */ - if (inst->mode != 0) - { - if (window_srcset) { - window_srcmin = min(window_srcmin, inst->addr); - window_srcmax = max(window_srcmax, inst->addr + take); - } else { - window_srcset = 1; - window_srcmin = inst->addr; - window_srcmax = inst->addr + take; - } - addr = inst->addr; - } - else - { - XD3_ASSERT (inst->addr >= window_start); - addr = inst->addr - window_start; - } - IF_DEBUG1 (DP(RINT "[merge copy] winpos %u take %u addr %"Q"u mode %u\n", - window_pos, take, addr, inst->mode)); - if ((ret = xd3_found_match (recode_stream, window_pos, take, - addr, inst->mode != 0))) - { - return ret; - } - break; - } - - window_pos += take; - output_pos += take; - - if (take == inst->size) - { - inst_pos += 1; - } - else - { - /* Modify the instruction for the next pass. */ - if (inst->type != XD3_RUN) - { - inst->addr += take; - } - inst->size -= take; - } - } - - xd3_avail_input (recode_stream, main_bdata, window_pos); - - recode_stream->enc_state = ENC_INSTR; - - if (window_srcset) { - recode_stream->srcwin_decided = 1; - recode_stream->src = &recode_source; - recode_source.srclen = window_srcmax - window_srcmin; - recode_source.srcbase = window_srcmin; - recode_stream->taroff = recode_source.srclen; - } else { - recode_stream->srcwin_decided = 0; - recode_stream->src = NULL; - recode_stream->taroff = 0; - } - - for (;;) - { - switch ((ret = xd3_encode_input (recode_stream))) - { - case XD3_INPUT: { - goto done_window; - } - case XD3_OUTPUT: { - /* main_file_write below */ - break; - } - case XD3_GOTHEADER: - case XD3_WINSTART: - case XD3_WINFINISH: { - /* ignore */ - continue; - } - case XD3_GETSRCBLK: - case 0: { - return XD3_INTERNAL; - } - default: - return ret; - } - - if ((ret = main_write_output(recode_stream, ofile))) - { - return ret; - } - - xd3_consume_output (recode_stream); - } - done_window: - (void) 0; - } - - return 0; -} -#endif - -/******************************************************************* - Input decompression, output recompression - ******************************************************************/ - -#if EXTERNAL_COMPRESSION -/* This is tricky POSIX-specific code with lots of fork(), pipe(), - * dup(), waitpid(), and exec() business. Most of this code - * originated in PRCS1, which did automatic package-file - * decompression. It works with both XD3_POSIX and XD3_STDIO file - * disciplines. - * - * To automatically detect compressed inputs requires a child process - * to reconstruct the input stream, which was advanced in order to - * detect compression, because it may not be seekable. In other - * words, the main program reads part of the input stream, and if it - * detects a compressed input it then forks a pipe copier process, - * which copies the first-read block out of the main-program's memory, - * then streams the remaining compressed input into the - * input-decompression pipe. - */ - -#include <unistd.h> -#include <sys/stat.h> -#include <sys/wait.h> - -/* Remember which pipe FD is which. */ -#define PIPE_READ_FD 0 -#define PIPE_WRITE_FD 1 - -static pid_t ext_subprocs[2]; -static char* ext_tmpfile = NULL; - -/* Like write(), but makes repeated calls to empty the buffer. */ -static int -main_pipe_write (int outfd, uint8_t *exist_buf, usize_t remain) -{ - int ret; - - if ((ret = xd3_posix_io (outfd, exist_buf, remain, - (xd3_posix_func*) &write, NULL))) - { - XPR(NT "pipe write failed: %s", xd3_mainerror (ret)); - return ret; - } - - return 0; -} - -/* A simple error-reporting waitpid interface. */ -static int -main_waitpid_check(pid_t pid) -{ - int status; - int ret = 0; - - if (waitpid (pid, & status, 0) < 0) - { - ret = get_errno (); - XPR(NT "compression subprocess: wait: %s\n", xd3_mainerror (ret)); - } - else if (! WIFEXITED (status)) - { - ret = ECHILD; - XPR(NT "compression subprocess: signal %d\n", - WIFSIGNALED (status) ? WTERMSIG (status) : WSTOPSIG (status)); - } - else if (WEXITSTATUS (status) != 0) - { - ret = ECHILD; - XPR(NT "compression subprocess: exit %d\n", WEXITSTATUS (status)); - } - - return ret; -} - -/* Wait for any existing child processes to check for abnormal exit. */ -static int -main_external_compression_finish (void) -{ - int i; - int ret; - - for (i = 0; i < 2; i += 1) - { - if (! ext_subprocs[i]) { continue; } - - if ((ret = main_waitpid_check (ext_subprocs[i]))) - { - return ret; - } - } - - return 0; -} - -/* This runs as a forked process of main_input_decompress_setup() to - * copy input to the decompression process. First, the available - * input is copied out of the existing buffer, then the buffer is - * reused to continue reading from the compressed input file. */ -static int -main_pipe_copier (uint8_t *pipe_buf, - usize_t pipe_bufsize, - usize_t nread, - main_file *ifile, - int outfd) -{ - int ret; - - for (;;) - { - if (nread > 0 && (ret = main_pipe_write (outfd, pipe_buf, nread))) - { - return ret; - } - - if (nread < pipe_bufsize) - { - break; - } - - if ((ret = main_file_read (ifile, pipe_buf, pipe_bufsize, - & nread, "pipe read failed")) < 0) - { - return ret; - } - } - - return 0; -} - -/* This function is called after we have read some amount of data from - * the input file and detected a compressed input. Here we start a - * decompression subprocess by forking twice. The first process runs - * the decompression command, the second process copies data to the - * input of the first. */ -static int -main_input_decompress_setup (const main_extcomp *decomp, - main_file *ifile, - uint8_t *input_buf, - usize_t input_bufsize, - uint8_t *pipe_buf, - usize_t pipe_bufsize, - usize_t pipe_avail, - usize_t *nread) -{ - /* The two pipes: input and output file descriptors. */ - int outpipefd[2], inpipefd[2]; - int input_fd = -1; /* The resulting input_fd (output of decompression). */ - pid_t decomp_id, copier_id; /* The two subprocs. */ - int ret; - - outpipefd[0] = outpipefd[1] = -1; - inpipefd[0] = inpipefd[1] = -1; - - if (pipe (outpipefd) || pipe (inpipefd)) - { - XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - - if ((decomp_id = fork ()) < 0) - { - XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - - /* The first child runs the decompression process: */ - if (decomp_id == 0) - { - /* Setup pipes: write to the outpipe, read from the inpipe. */ - if (dup2 (outpipefd[PIPE_WRITE_FD], STDOUT_FILENO) < 0 || - dup2 (inpipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || - close (outpipefd[PIPE_READ_FD]) || - close (outpipefd[PIPE_WRITE_FD]) || - close (inpipefd[PIPE_READ_FD]) || - close (inpipefd[PIPE_WRITE_FD]) || - execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, - decomp->decomp_options, NULL)) - { - XPR(NT "child process %s failed to execute: %s\n", - decomp->decomp_cmdname, xd3_mainerror (get_errno ())); - } - - _exit (127); - } - - ext_subprocs[0] = decomp_id; - - if ((copier_id = fork ()) < 0) - { - XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - - /* The second child runs the copier process: */ - if (copier_id == 0) - { - int exitval = 0; - - if (close (inpipefd[PIPE_READ_FD]) || - main_pipe_copier (pipe_buf, pipe_bufsize, pipe_avail, - ifile, inpipefd[PIPE_WRITE_FD]) || - close (inpipefd[PIPE_WRITE_FD])) - { - XPR(NT "child copier process failed: %s\n", - xd3_mainerror (get_errno ())); - exitval = 1; - } - - _exit (exitval); - } - - ext_subprocs[1] = copier_id; - - /* The parent closes both pipes after duplicating the output of - * compression. */ - input_fd = dup (outpipefd[PIPE_READ_FD]); - - if (input_fd < 0 || - main_file_close (ifile) || - close (outpipefd[PIPE_READ_FD]) || - close (outpipefd[PIPE_WRITE_FD]) || - close (inpipefd[PIPE_READ_FD]) || - close (inpipefd[PIPE_WRITE_FD])) - { - XPR(NT "dup/close failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#if XD3_STDIO - /* Note: fdopen() acquires the fd, closes it when finished. */ - if ((ifile->file = fdopen (input_fd, "r")) == NULL) - { - XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#elif XD3_POSIX - ifile->file = input_fd; -#endif - - ifile->compressor = decomp; - - /* Now the input file is decompressed. */ - return main_file_read (ifile, input_buf, input_bufsize, - nread, "input decompression failed"); - - pipe_cleanup: - close (input_fd); - close (outpipefd[PIPE_READ_FD]); - close (outpipefd[PIPE_WRITE_FD]); - close (inpipefd[PIPE_READ_FD]); - close (inpipefd[PIPE_WRITE_FD]); - return ret; -} - - -/* This routine is called when the first buffer of input data is read - * by the main program (unless input decompression is disabled by - * command-line option). If it recognizes the magic number of a known - * input type it invokes decompression. - * - * Skips decompression if the decompression type or the file type is - * RD_NONEXTERNAL. - * - * Behaves exactly like main_file_read, otherwise. - * - * This function uses a separate buffer to read the first small block - * of input. If a compressed input is detected, the separate buffer - * is passed to the pipe copier. This avoids using the same size - * buffer in both cases. */ -static int -main_decompress_input_check (main_file *ifile, - uint8_t *input_buf, - usize_t input_size, - usize_t *nread) -{ - int ret; - usize_t i; - usize_t check_nread; - uint8_t check_buf[XD3_ALLOCSIZE]; - - if ((ret = main_file_read (ifile, check_buf, - min (input_size, XD3_ALLOCSIZE), - & check_nread, "input read failed"))) - { - return ret; - } - - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) - { - const main_extcomp *decomp = & extcomp_types[i]; - - if ((check_nread > decomp->magic_size) && - /* The following expr skips decompression if we are trying - * to read a VCDIFF input and that is the magic number. */ - !((decomp->flags & RD_NONEXTERNAL) && - (ifile->flags & RD_NONEXTERNAL)) && - memcmp (check_buf, decomp->magic, decomp->magic_size) == 0) - { - if (! option_quiet) - { - XPR(NT "%s | %s %s\n", - ifile->filename, - decomp->decomp_cmdname, - decomp->decomp_options); - } - - return main_input_decompress_setup (decomp, ifile, - input_buf, input_size, - check_buf, XD3_ALLOCSIZE, - check_nread, nread); - } - } - - /* Now read the rest of the input block. */ - (*nread) = 0; - - if (check_nread == XD3_ALLOCSIZE) - { - ret = main_file_read (ifile, input_buf + XD3_ALLOCSIZE, - input_size - XD3_ALLOCSIZE, nread, - "input read failed"); - } - - memcpy (input_buf, check_buf, check_nread); - - (*nread) += check_nread; - - return 0; -} - -/* This is called when the source file needs to be decompressed. We - * fork/exec a decompression command with the proper input and output - * to a temporary file. */ -static int -main_decompress_source (main_file *sfile, xd3_source *source) -{ - const main_extcomp *decomp = sfile->compressor; - pid_t decomp_id; /* One subproc. */ - int input_fd = -1; - int output_fd = -1; - int ret; - char *tmpname = NULL; - char *tmpdir = getenv ("TMPDIR"); - static const char tmpl[] = "/xd3src.XXXXXX"; - - /* Make a template for mkstmp() */ - if (tmpdir == NULL) { tmpdir = "/tmp"; } - if ((tmpname = - (char*) main_malloc (strlen (tmpdir) + sizeof (tmpl) + 1)) == NULL) - { - return ENOMEM; - } - sprintf (tmpname, "%s%s", tmpdir, tmpl); - - XD3_ASSERT (ext_tmpfile == NULL); - ext_tmpfile = tmpname; - - /* Open the output FD. */ - if ((output_fd = mkstemp (tmpname)) < 0) - { - XPR(NT "mkstemp failed: %s: %s", - tmpname, xd3_mainerror (ret = get_errno ())); - goto cleanup; - } - - /* Copy the input FD, reset file position. */ - XD3_ASSERT (main_file_isopen (sfile)); -#if XD3_STDIO - if ((input_fd = dup (fileno (sfile->file))) < 0) - { - XPR(NT "dup failed: %s", xd3_mainerror (ret = get_errno ())); - goto cleanup; - } - main_file_close (sfile); - sfile->file = NULL; -#elif XD3_POSIX - input_fd = sfile->file; - sfile->file = -1; -#endif - - if ((ret = lseek (input_fd, SEEK_SET, 0)) != 0) - { - XPR(NT "lseek failed: : %s", xd3_mainerror (ret = get_errno ())); - goto cleanup; - } - - if ((decomp_id = fork ()) < 0) - { - XPR(NT "fork failed: %s", xd3_mainerror (ret = get_errno ())); - goto cleanup; - } - - /* The child runs the decompression process: */ - if (decomp_id == 0) - { - /* Setup pipes: write to the output file, read from the pipe. */ - if (dup2 (input_fd, STDIN_FILENO) < 0 || - dup2 (output_fd, STDOUT_FILENO) < 0 || - execlp (decomp->decomp_cmdname, decomp->decomp_cmdname, - decomp->decomp_options, NULL)) - { - XPR(NT "child process %s failed to execute: %s\n", - decomp->decomp_cmdname, xd3_mainerror (get_errno ())); - } - - _exit (127); - } - - close (input_fd); - close (output_fd); - input_fd = -1; - output_fd = -1; - - /* Then wait for completion. */ - if ((ret = main_waitpid_check (decomp_id))) - { - goto cleanup; - } - - /* Open/stat the decompressed source file. */ - if ((ret = main_file_open (sfile, tmpname, XO_READ))) { goto cleanup; } - if ((ret = main_file_stat (sfile, & source->size, 1))) { goto cleanup; } - return 0; - - cleanup: - close (input_fd); - close (output_fd); - if (tmpname) { free (tmpname); } - ext_tmpfile = NULL; - return ret; -} - -/* Initiate re-compression of the output stream. This is easier than - * input decompression because we know beforehand that the stream will - * be compressed, whereas the input has already been read when we - * decide it should be decompressed. Thus, it only requires one - * subprocess and one pipe. */ -static int -main_recompress_output (main_file *ofile) -{ - pid_t recomp_id; /* One subproc. */ - int pipefd[2]; /* One pipe. */ - int output_fd = -1; - int ret; - const main_extcomp *recomp = ofile->compressor; - - pipefd[0] = pipefd[1] = -1; - - if (pipe (pipefd)) - { - XPR(NT "pipe failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - - if ((recomp_id = fork ()) < 0) - { - XPR(NT "fork failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - - /* The child runs the recompression process: */ - if (recomp_id == 0) - { - /* Setup pipes: write to the output file, read from the pipe. */ - if (dup2 (XFNO (ofile), STDOUT_FILENO) < 0 || - dup2 (pipefd[PIPE_READ_FD], STDIN_FILENO) < 0 || - close (pipefd[PIPE_READ_FD]) || - close (pipefd[PIPE_WRITE_FD]) || - execlp (recomp->recomp_cmdname, recomp->recomp_cmdname, - recomp->recomp_options, NULL)) - { - XPR(NT "child process %s failed to execute: %s\n", - recomp->recomp_cmdname, xd3_mainerror (get_errno ())); - } - - _exit (127); - } - - ext_subprocs[0] = recomp_id; - - /* The parent closes both pipes after duplicating the output-fd for - * writing to the compression pipe. */ - output_fd = dup (pipefd[PIPE_WRITE_FD]); - - if (output_fd < 0 || - main_file_close (ofile) || - close (pipefd[PIPE_READ_FD]) || - close (pipefd[PIPE_WRITE_FD])) - { - XPR(NT "close failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#if XD3_STDIO - /* Note: fdopen() acquires the fd, closes it when finished. */ - if ((ofile->file = fdopen (output_fd, "w")) == NULL) - { - XPR(NT "fdopen failed: %s\n", xd3_mainerror (ret = get_errno ())); - goto pipe_cleanup; - } - -#elif XD3_POSIX - ofile->file = output_fd; -#endif - - /* Now the output file will be compressed. */ - return 0; - - pipe_cleanup: - close (output_fd); - close (pipefd[PIPE_READ_FD]); - close (pipefd[PIPE_WRITE_FD]); - return ret; -} -#endif /* EXTERNAL_COMPRESSION */ - -/* Identify the compressor that was used based on its ident string, - * which is passed in the application header. */ -static const main_extcomp* -main_ident_compressor (const char *ident) -{ - usize_t i; - - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) - { - if (strcmp (extcomp_types[i].ident, ident) == 0) - { - return & extcomp_types[i]; - } - } - - return NULL; -} - -/* Return the main_extcomp record to use for this identifier, if possible. */ -static const main_extcomp* -main_get_compressor (const char *ident) -{ - const main_extcomp *ext = main_ident_compressor (ident); - - if (ext == NULL) - { - if (! option_quiet) - { - XPR(NT "warning: cannot recompress output: " - "unrecognized external compression ID: %s\n", ident); - } - return NULL; - } - else if (! EXTERNAL_COMPRESSION) - { - if (! option_quiet) - { - XPR(NT "warning: external support not compiled: " - "original input was compressed: %s\n", ext->recomp_cmdname); - } - return NULL; - } - else - { - return ext; - } -} - -/********************************************************************* - APPLICATION HEADER - *******************************************************************/ - -#if XD3_ENCODER -static const char* -main_apphead_string (const char* x) -{ - const char *y; - - if (x == NULL) { return ""; } - - if (strcmp (x, "/dev/stdin") == 0 || - strcmp (x, "/dev/stdout") == 0 || - strcmp (x, "/dev/stderr") == 0) { return "-"; } - - // TODO: this is not portable - return (y = strrchr (x, '/')) == NULL ? x : y + 1; -} - -static int -main_set_appheader (xd3_stream *stream, main_file *input, main_file *sfile) -{ - /* The user may disable the application header. Once the appheader - * is set, this disables setting it again. */ - if (appheader_used || ! option_use_appheader) { return 0; } - - /* The user may specify the application header, otherwise format the - default header. */ - if (option_appheader) - { - appheader_used = option_appheader; - } - else - { - const char *iname; - const char *icomp; - const char *sname; - const char *scomp; - int len; - - iname = main_apphead_string (input->filename); - icomp = (input->compressor == NULL) ? "" : input->compressor->ident; - len = strlen (iname) + strlen (icomp) + 2; - - if (sfile->filename != NULL) - { - sname = main_apphead_string (sfile->filename); - scomp = (sfile->compressor == NULL) ? "" : sfile->compressor->ident; - len += strlen (sname) + strlen (scomp) + 2; - } - else - { - sname = scomp = ""; - } - - if ((appheader_used = (uint8_t*) main_malloc (len)) == NULL) - { - return ENOMEM; - } - - if (sfile->filename == NULL) - { - sprintf ((char*)appheader_used, "%s/%s", iname, icomp); - } - else - { - sprintf ((char*)appheader_used, "%s/%s/%s/%s", - iname, icomp, sname, scomp); - } - } - - xd3_set_appheader (stream, appheader_used, strlen ((char*)appheader_used)); - - return 0; -} -#endif - -static void -main_get_appheader_params (main_file *file, char **parsed, - int output, const char *type, - main_file *other) -{ - /* Set the filename if it was not specified. If output, option_stdout (-c) - * overrides. */ - if (file->filename == NULL && - ! (output && option_stdout) && - strcmp (parsed[0], "-") != 0) - { - file->filename = parsed[0]; - - if (other->filename != NULL) { - /* Take directory from the other file, if it has one. */ - /* TODO: This results in nonsense names like /dev/foo.tar.gz - * and probably the filename-default logic interferes with - * multi-file operation and the standard file extension? - * Possibly the name header is bad, should be off by default. - * Possibly we just want to remember external/compression - * settings. */ - char *last_slash = strrchr(other->filename, '/'); - - if (last_slash != NULL) { - int dlen = last_slash - other->filename; - - XD3_ASSERT(file->filename_copy == NULL); - file->filename_copy = - (char*) main_malloc(dlen + 2 + strlen(file->filename)); - - strncpy(file->filename_copy, other->filename, dlen); - file->filename_copy[dlen] = '/'; - strcpy(file->filename_copy + dlen + 1, parsed[0]); - - file->filename = file->filename_copy; - } - } - - if (! option_quiet) - { - XPR(NT "using default %s filename: %s\n", type, file->filename); - } - } - - /* Set the compressor, initiate de/recompression later. */ - if (file->compressor == NULL && *parsed[1] != 0) - { - file->compressor = main_get_compressor (parsed[1]); - } -} - -static void -main_get_appheader (xd3_stream *stream, main_file *ifile, - main_file *output, main_file *sfile) -{ - uint8_t *apphead; - usize_t appheadsz; - int ret; - - /* The user may disable the application header. Once the appheader - * is set, this disables setting it again. */ - if (! option_use_appheader) { return; } - - ret = xd3_get_appheader (stream, & apphead, & appheadsz); - - /* Ignore failure, it only means we haven't received a header yet. */ - if (ret != 0) { return; } - - if (appheadsz > 0) - { - char *start = (char*)apphead; - char *slash; - int place = 0; - char *parsed[4]; - - memset (parsed, 0, sizeof (parsed)); - - while ((slash = strchr (start, '/')) != NULL) - { - *slash = 0; - parsed[place++] = start; - start = slash + 1; - } - - parsed[place++] = start; - - /* First take the output parameters. */ - if (place == 2 || place == 4) - { - main_get_appheader_params (output, parsed, 1, "output", ifile); - } - - /* Then take the source parameters. */ - if (place == 4) - { - main_get_appheader_params (sfile, parsed+2, 0, "source", ifile); - } - } - - option_use_appheader = 0; - return; -} - -/********************************************************************* - Main I/O routines - **********************************************************************/ - -/* This function acts like the above except it may also try to - * recognize a compressed input when the first buffer of data is read. - * The EXTERNAL_COMPRESSION code is called to search for magic - * numbers. */ -static int -main_read_primary_input (main_file *ifile, - uint8_t *buf, - usize_t size, - usize_t *nread) -{ -#if EXTERNAL_COMPRESSION - if (option_decompress_inputs && ifile->flags & RD_FIRST) - { - ifile->flags &= ~RD_FIRST; - - return main_decompress_input_check (ifile, buf, size, nread); - } -#endif - - return main_file_read (ifile, buf, size, nread, "input read failed"); -} - -/* Open the main output file, sets a default file name, initiate - * recompression. This function is expected to fprint any error - * messages. */ -static int -main_open_output (xd3_stream *stream, main_file *ofile) -{ - int ret; - - if (option_no_output) - { - return 0; - } - - if (ofile->filename == NULL) - { - XSTDOUT_XF (ofile); - - if (option_verbose > 1) - { - XPR(NT "using standard output: %s\n", ofile->filename); - } - } - else - { - /* Stat the file to check for overwrite. */ - if (option_force == 0 && main_file_exists (ofile)) - { - if (!option_quiet) - { - XPR(NT "to overwrite output file specify -f: %s\n", - ofile->filename); - } - return EEXIST; - } - - if ((ret = main_file_open (ofile, ofile->filename, XO_WRITE))) - { - return ret; - } - - if (option_verbose > 1) { XPR(NT "output file: %s\n", ofile->filename); } - } - -#if EXTERNAL_COMPRESSION - /* Do output recompression. */ - if (ofile->compressor != NULL && option_recompress_outputs == 1) - { - if (! option_quiet) - { - XPR(NT "%s %s | %s\n", - ofile->compressor->recomp_cmdname, - ofile->compressor->recomp_options, - ofile->filename); - } - - if ((ret = main_recompress_output (ofile))) - { - return ret; - } - } -#endif - - return 0; -} - -/* This is called at different times for encoding and decoding. The - * encoder calls it immediately, the decoder delays until the - * application header is received. Stream may be NULL, in which case - * xd3_set_source is not called. */ -static int -main_set_source (xd3_stream *stream, int cmd, - main_file *sfile, xd3_source *source) -{ - int ret = 0; - usize_t i; - uint8_t *tmp_buf = NULL; - - /* Open it, check for seekability, set required xd3_source fields. */ - if (allow_fake_source) - { - sfile->mode = XO_READ; - sfile->realname = sfile->filename; - sfile->nread = 0; - source->size = XOFF_T_MAX; - } - else - { - if ((ret = main_file_open (sfile, sfile->filename, XO_READ)) || - (ret = main_file_stat (sfile, & source->size, 1))) - { - goto error; - } - } - - source->name = sfile->filename; - source->ioh = sfile; - source->curblkno = (xoff_t) -1; - source->curblk = NULL; - -#if EXTERNAL_COMPRESSION - if (option_decompress_inputs) - { - /* If encoding, read the header to check for decompression. */ - if (IS_ENCODE (cmd)) - { - usize_t nread; - tmp_buf = (uint8_t*) main_malloc (XD3_ALLOCSIZE); - - if ((ret = main_file_read (sfile, tmp_buf, XD3_ALLOCSIZE, - & nread, "source read failed"))) - { - goto error; - } - - /* Check known magic numbers. */ - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) - { - const main_extcomp *decomp = & extcomp_types[i]; - - if ((nread > decomp->magic_size) && - memcmp (tmp_buf, decomp->magic, decomp->magic_size) == 0) - { - sfile->compressor = decomp; - break; - } - } - - if (sfile->compressor == NULL) - { - if (option_verbose > 2) - { - XPR(NT "source block 0 read (not compressed)\n"); - } - } - } - - /* In either the encoder or decoder, start decompression. */ - if (sfile->compressor) - { - xoff_t osize = source->size; - - if ((ret = main_decompress_source (sfile, source))) - { - goto error; - } - - if (! option_quiet) - { - char s1[32], s2[32]; - XPR(NT "%s | %s %s => %s %.1f%% [ %s , %s ]\n", - sfile->filename, - sfile->compressor->decomp_cmdname, - sfile->compressor->decomp_options, - sfile->realname, - 100.0 * source->size / osize, - main_format_bcnt (osize, s1), - main_format_bcnt (source->size, s2)); - } - } - } -#endif - - /* At this point we know source->size. - * Source buffer, blksize, LRU init. */ - if (source->size < option_srcwinsz) - { - /* Reduce sizes to actual source size, read whole file */ - option_srcwinsz = source->size; - source->blksize = source->size; - lru_size = 1; - } - else - { - option_srcwinsz = max(option_srcwinsz, XD3_MINSRCWINSZ); - - source->blksize = (option_srcwinsz / LRU_SIZE); - lru_size = LRU_SIZE; - } - - main_blklru_list_init (& lru_list); - main_blklru_list_init (& lru_free); - - if (option_verbose) - { - static char buf[32]; - - XPR(NT "source %s winsize %s size %"Q"u\n", - sfile->filename, main_format_bcnt(option_srcwinsz, buf), - source->size); - } - - if (option_verbose > 1) - { - XPR(NT "source block size: %u\n", source->blksize); - } - - if ((lru = (main_blklru*) - main_malloc (sizeof (main_blklru) * lru_size)) == NULL) - { - ret = ENOMEM; - goto error; - } - - for (i = 0; i < lru_size; i += 1) - { - lru[i].blkno = (xoff_t) -1; - - if ((lru[i].blk = (uint8_t*) main_malloc (source->blksize)) == NULL) - { - ret = ENOMEM; - goto error; - } - - main_blklru_list_push_back (& lru_free, & lru[i]); - } - - if (stream && (ret = xd3_set_source (stream, source))) - { - XPR(NT XD3_LIB_ERRMSG (stream, ret)); - goto error; - } - - error: - if (tmp_buf != NULL) - { - main_free (tmp_buf); - } - - return ret; -} - -static usize_t -main_get_winsize (main_file *ifile) { - xoff_t file_size; - usize_t size = option_winsize; - - if (main_file_stat (ifile, &file_size, 0) == 0) - { - size = (usize_t) min(file_size, (xoff_t) size); - } - - size = max(size, XD3_ALLOCSIZE); - - if (option_verbose > 1) - { - XPR(NT "input window size: %u\n", size); - } - - return size; -} - -/******************************************************************* - Source routines - *******************************************************************/ - -/* This is the callback for reading a block of source. This function - * is blocking and it implements a small LRU. - * - * Note that it is possible for main_input() to handle getblk requests - * in a non-blocking manner. If the callback is NULL then the caller - * of xd3_*_input() must handle the XD3_GETSRCBLK return value and - * fill the source in the same way. See xd3_getblk for details. To - * see an example of non-blocking getblk, see xdelta-test.h. */ -static int -main_getblk_func (xd3_stream *stream, - xd3_source *source, - xoff_t blkno) -{ - int ret; - xoff_t pos = blkno * source->blksize; - main_file *sfile = (main_file*) source->ioh; - main_blklru *blru = NULL; - usize_t onblk = xd3_bytes_on_srcblk_fast (source, blkno); - usize_t nread; - usize_t i; - - if (allow_fake_source) - { - source->curblkno = blkno; - source->onblk = onblk; - source->curblk = lru[0].blk; - return 0; - } - - if (do_not_lru) - { - /* Direct lookup assumes sequential scan w/o skipping blocks. */ - int idx = blkno % lru_size; - if (lru[idx].blkno == blkno) - { - source->curblkno = blkno; - source->onblk = onblk; - source->curblk = lru[idx].blk; - lru_hits += 1; - return 0; - } - - if (lru[idx].blkno != (xoff_t)-1 && - lru[idx].blkno != (xoff_t)(blkno - lru_size)) - { - return XD3_TOOFARBACK; - } - } - else - { - /* Sequential search through LRU. */ - for (i = 0; i < lru_size; i += 1) - { - if (lru[i].blkno == blkno) - { - main_blklru_list_remove (& lru[i]); - main_blklru_list_push_back (& lru_list, & lru[i]); - - source->curblkno = blkno; - source->onblk = onblk; - source->curblk = lru[i].blk; - lru_hits += 1; - return 0; - } - } - } - - if (! main_blklru_list_empty (& lru_free)) - { - blru = main_blklru_list_pop_front (& lru_free); - } - else if (! main_blklru_list_empty (& lru_list)) - { - if (do_not_lru) { - blru = & lru[blkno % lru_size]; - main_blklru_list_remove(blru); - } else { - blru = main_blklru_list_pop_front (& lru_list); - } - lru_misses += 1; - } - - lru_filled += 1; - - if ((ret = main_file_seek (sfile, pos))) - { - return ret; - } - - if ((ret = main_file_read (sfile, (uint8_t*) blru->blk, source->blksize, - & nread, "source read failed"))) - { - return ret; - } - - if (nread != onblk) - { - XPR(NT "source file size change: %s\n", sfile->filename); - return XD3_INTERNAL; - } - - main_blklru_list_push_back (& lru_list, blru); - - if (option_verbose > 3) - { - if (blru->blkno != (xoff_t)-1) - { - XPR(NT "source block %"Q"u ejects %"Q"u (lru_hits=%u, " - "lru_misses=%u, lru_filled=%u)\n", - blkno, blru->blkno, lru_hits, lru_misses, lru_filled); - } - else - { - XPR(NT "source block %"Q"u read (lru_hits=%u, lru_misses=%u, " - "lru_filled=%u)\n", blkno, lru_hits, lru_misses, lru_filled); - } - } - - blru->blkno = blkno; - source->curblk = blru->blk; - source->curblkno = blkno; - source->onblk = onblk; - - return 0; -} - -/********************************************************************* - Main routines - ********************************************************************/ - -/* This is a generic input function. It calls the xd3_encode_input or - * xd3_decode_input functions and makes calls to the various input - * handling routines above, which coordinate external decompression. - */ -static int -main_input (xd3_cmd cmd, - main_file *ifile, - main_file *ofile, - main_file *sfile) -{ - int ret; - xd3_stream stream; - usize_t nread; - usize_t winsize; - int stream_flags = 0; - xd3_config config; - xd3_source source; - xoff_t last_total_in = 0; - xoff_t last_total_out = 0; - long start_time; - int stdout_only = 0; - int (*input_func) (xd3_stream*); - int (*output_func) (xd3_stream*, main_file *); - - memset (& stream, 0, sizeof (stream)); - memset (& source, 0, sizeof (source)); - memset (& config, 0, sizeof (config)); - - config.alloc = main_alloc; - config.freef = main_free1; - - config.iopt_size = option_iopt_size; - config.sprevsz = option_sprevsz; - - do_not_lru = 0; - - start_time = get_millisecs_now (); - - if (option_use_checksum) { stream_flags |= XD3_ADLER32; } - - /* main_input setup. */ - switch ((int) cmd) - { -#if VCDIFF_TOOLS - if (1) { case CMD_PRINTHDR: stream_flags |= XD3_JUST_HDR; } - else if (1) { case CMD_PRINTHDRS: stream_flags |= XD3_SKIP_WINDOW; } - else { case CMD_PRINTDELTA: stream_flags |= XD3_SKIP_EMIT; } - ifile->flags |= RD_NONEXTERNAL; - input_func = xd3_decode_input; - output_func = main_print_func; - stream_flags |= XD3_ADLER32_NOVER; - stdout_only = 1; - break; - - case CMD_RECODE: - case CMD_MERGE: - case CMD_MERGE_ARG: - /* No source will be read */ - stream_flags |= XD3_ADLER32_NOVER | XD3_SKIP_EMIT; - ifile->flags |= RD_NONEXTERNAL; - input_func = xd3_decode_input; - - if ((ret = main_init_recode_stream ())) - { - return EXIT_FAILURE; - } - - if (cmd == CMD_RECODE) { output_func = main_recode_func; } - else { output_func = main_merge_func; } - break; -#endif /* VCDIFF_TOOLS */ - -#if XD3_ENCODER - case CMD_ENCODE: - do_not_lru = 1; - input_func = xd3_encode_input; - output_func = main_write_output; - - if (option_no_compress) { stream_flags |= XD3_NOCOMPRESS; } - if (option_use_altcodetable) { stream_flags |= XD3_ALT_CODE_TABLE; } - if (option_smatch_config) - { - char *s = option_smatch_config, *e; - int values[XD3_SOFTCFG_VARCNT]; - int got; - - config.smatch_cfg = XD3_SMATCH_SOFT; - - for (got = 0; got < XD3_SOFTCFG_VARCNT; got += 1, s = e + 1) - { - values[got] = strtol (s, &e, 10); - - if ((values[got] < 0) || - (e == s) || - (got < XD3_SOFTCFG_VARCNT-1 && *e == 0) || - (got == XD3_SOFTCFG_VARCNT-1 && *e != 0)) - { - XPR(NT "invalid string match specifier (-C) %d: %s\n", - got, s); - return EXIT_FAILURE; - } - } - - config.smatcher_soft.large_look = values[0]; - config.smatcher_soft.large_step = values[1]; - config.smatcher_soft.small_look = values[2]; - config.smatcher_soft.small_chain = values[3]; - config.smatcher_soft.small_lchain = values[4]; - config.smatcher_soft.max_lazy = values[5]; - config.smatcher_soft.long_enough = values[6]; - } - else - { - if (option_verbose > 1) - { - XPR(NT "compression level: %d\n", option_level); - } - if (option_level == 0) - { - stream_flags |= XD3_NOCOMPRESS; - config.smatch_cfg = XD3_SMATCH_FASTEST; - } - else if (option_level == 1) - { config.smatch_cfg = XD3_SMATCH_FASTEST; } - else if (option_level == 2) - { config.smatch_cfg = XD3_SMATCH_FASTER; } - else if (option_level <= 5) - { config.smatch_cfg = XD3_SMATCH_FAST; } - else if (option_level == 6) - { config.smatch_cfg = XD3_SMATCH_DEFAULT; } - else - { config.smatch_cfg = XD3_SMATCH_SLOW; } - } - break; -#endif - case CMD_DECODE: - if (option_use_checksum == 0) { stream_flags |= XD3_ADLER32_NOVER; } - ifile->flags |= RD_NONEXTERNAL; - input_func = xd3_decode_input; - output_func = main_write_output; - break; - default: - XPR(NT "internal error\n"); - return EXIT_FAILURE; - } - - main_bsize = winsize = main_get_winsize (ifile); - - if ((main_bdata = (uint8_t*) main_malloc (winsize)) == NULL) - { - return EXIT_FAILURE; - } - - if (IS_ENCODE (cmd)) - { - /* When encoding, open the source file, possibly decompress it. - * The decoder delays this step until XD3_GOTHEADER. */ - if (sfile->filename != NULL && - (ret = main_set_source (NULL, cmd, sfile, & source))) - { - return EXIT_FAILURE; - } - } - - config.winsize = winsize; - config.srcwin_maxsz = option_srcwinsz; - config.getblk = main_getblk_func; - config.flags = stream_flags; - - if ((ret = main_set_secondary_flags (&config)) || - (ret = xd3_config_stream (& stream, & config))) - { - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } - -#if VCDIFF_TOOLS - if ((cmd == CMD_MERGE || cmd == CMD_MERGE_ARG) && - (ret = xd3_whole_state_init (& stream))) - { - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } -#endif - - if (IS_ENCODE (cmd) && sfile->filename != NULL && - (ret = xd3_set_source (& stream, & source))) - { - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } - - /* This times each window. */ - get_millisecs_since (); - - /* Main input loop. */ - do - { - xoff_t input_offset; - xoff_t input_remain; - usize_t try_read; - - input_offset = ifile->nread; - - input_remain = XOFF_T_MAX - input_offset; - - try_read = (usize_t) min ((xoff_t) config.winsize, input_remain); - - if ((ret = main_read_primary_input (ifile, main_bdata, - try_read, & nread))) - { - return EXIT_FAILURE; - } - - /* If we've reached EOF tell the stream to flush. */ - if (nread < try_read) - { - stream.flags |= XD3_FLUSH; - } - -#if XD3_ENCODER - /* After the first main_read_primary_input completes, we know - * all the information needed to encode the application - * header. */ - if (cmd == CMD_ENCODE && - (ret = main_set_appheader (& stream, ifile, sfile))) - { - return EXIT_FAILURE; - } -#endif - xd3_avail_input (& stream, main_bdata, nread); - - /* If we read zero bytes after encoding at least one window... */ - if (nread == 0 && stream.current_window > 0) { - break; - } - - again: - ret = input_func (& stream); - - switch (ret) - { - case XD3_INPUT: - continue; - - case XD3_GOTHEADER: - { - XD3_ASSERT (stream.current_window == 0); - - /* Need to process the appheader as soon as possible. It may - * contain a suggested default filename/decompression routine for - * the ofile, and it may contain default/decompression routine for - * the sources. */ - if (cmd == CMD_DECODE) - { - /* May need to set the sfile->filename if none was given. */ - main_get_appheader (& stream, ifile, ofile, sfile); - - /* Now open the source file. */ - if ((sfile->filename != NULL) && - (ret = main_set_source (& stream, cmd, sfile, & source))) - { - return EXIT_FAILURE; - } - } - else if (cmd == CMD_PRINTHDR || - cmd == CMD_PRINTHDRS || - cmd == CMD_PRINTDELTA || - cmd == CMD_RECODE) - { - if (sfile->filename == NULL) - { - allow_fake_source = 1; - sfile->filename = "<placeholder>"; - main_set_source (& stream, cmd, sfile, & source); - } - } - } - /* FALLTHROUGH */ - case XD3_WINSTART: - { - /* e.g., set or unset XD3_SKIP_WINDOW. */ - goto again; - } - - case XD3_OUTPUT: - { - /* Defer opening the output file until the stream produces its - * first output for both encoder and decoder, this way we - * delay long enough for the decoder to receive the - * application header. (Or longer if there are skipped - * windows, but I can't think of any reason not to delay - * open.) */ - if (ofile != NULL && - ! main_file_isopen (ofile) && - (ret = main_open_output (& stream, ofile)) != 0) - { - return EXIT_FAILURE; - } - - if ((ret = output_func (& stream, ofile)) && - (ret != PRINTHDR_SPECIAL)) - { - return EXIT_FAILURE; - } - - if (ret == PRINTHDR_SPECIAL) - { - xd3_abort_stream (& stream); - ret = EXIT_SUCCESS; - goto done; - } - - ret = 0; - - xd3_consume_output (& stream); - goto again; - } - - case XD3_WINFINISH: - { - if (IS_ENCODE (cmd) || cmd == CMD_DECODE || cmd == CMD_RECODE) - { - if (! option_quiet && IS_ENCODE (cmd) && - main_file_isopen (sfile)) - { - /* Warn when no source copies are found */ - if (option_verbose && ! xd3_encoder_used_source (& stream)) - { - XPR(NT "warning: input window %"Q"u..%"Q"u has " - "no source copies\n", - stream.current_window * winsize, - (stream.current_window+1) * winsize); - } - - /* Limited i-buffer size affects source copies */ - if (option_verbose > 1 && - stream.i_slots_used > stream.iopt_size) - { - XPR(NT "warning: input position %"Q"u overflowed " - "instruction buffer, needed %u (vs. %u), " - "consider raising -I\n", - stream.current_window * winsize, - stream.i_slots_used, stream.iopt_size); - } - } - - if (option_verbose) - { - char rrateavg[32], wrateavg[32], tm[32]; - char rdb[32], wdb[32]; - char trdb[32], twdb[32]; - long millis = get_millisecs_since (); - usize_t this_read = (usize_t)(stream.total_in - - last_total_in); - usize_t this_write = (usize_t)(stream.total_out - - last_total_out); - last_total_in = stream.total_in; - last_total_out = stream.total_out; - - if (option_verbose > 1) - { - XPR(NT "%"Q"u: in %s (%s): out %s (%s): " - "total in %s: out %s: %s\n", - stream.current_window, - main_format_bcnt (this_read, rdb), - main_format_rate (this_read, millis, rrateavg), - main_format_bcnt (this_write, wdb), - main_format_rate (this_write, millis, wrateavg), - main_format_bcnt (stream.total_in, trdb), - main_format_bcnt (stream.total_out, twdb), - main_format_millis (millis, tm)); - } - else - { - XPR(NT "%"Q"u: in %s: out %s: total in %s: " - "out %s: %s\n", - stream.current_window, - main_format_bcnt (this_read, rdb), - main_format_bcnt (this_write, wdb), - main_format_bcnt (stream.total_in, trdb), - main_format_bcnt (stream.total_out, twdb), - main_format_millis (millis, tm)); - } - } - } - goto again; - } - - default: - /* input_func() error */ - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } - } - while (nread == config.winsize); -done: - /* Close the inputs. (ifile must be open, sfile may be open) */ - main_file_close (ifile); - if (sfile != NULL) - { - main_file_close (sfile); - } - -#if VCDIFF_TOOLS - if (cmd == CMD_MERGE && - (ret = main_merge_output (& stream, ofile))) - { - return EXIT_FAILURE; - } - - if (cmd == CMD_MERGE_ARG) - { - xd3_swap_whole_state (& stream.whole_target, - & recode_stream->whole_target); - } -#endif /* VCDIFF_TOOLS */ - - /* If output file is not open yet because of delayed-open, it means - * we never encountered a window in the delta, but it could have had - * a VCDIFF header? TODO: solve this elsewhere. For now, it prints - * "nothing to output" below, but the check doesn't happen in case - * of option_no_output. */ - if (! option_no_output && ofile != NULL) - { - if (!stdout_only && ! main_file_isopen (ofile)) - { - XPR(NT "nothing to output: %s\n", ifile->filename); - return EXIT_FAILURE; - } - - /* Have to close the output before calling - * main_external_compression_finish, or else it hangs. */ - if (main_file_close (ofile) != 0) - { - return EXIT_FAILURE; - } - } - -#if EXTERNAL_COMPRESSION - if ((ret = main_external_compression_finish ())) - { - XPR(NT "external compression commands failed\n"); - return EXIT_FAILURE; - } -#endif - - if ((ret = xd3_close_stream (& stream))) - { - XPR(NT XD3_LIB_ERRMSG (& stream, ret)); - return EXIT_FAILURE; - } - -#if XD3_ENCODER - if (option_verbose > 1 && cmd == CMD_ENCODE) - { - XPR(NT "scanner configuration: %s\n", stream.smatcher.name); - XPR(NT "target hash table size: %u\n", stream.small_hash.size); - if (sfile != NULL && sfile->filename != NULL) - { - XPR(NT "source hash table size: %u\n", stream.large_hash.size); - } - } - - if (option_verbose > 2 && cmd == CMD_ENCODE) - { - XPR(NT "source copies: %"Q"u (%"Q"u bytes)\n", - stream.n_scpy, stream.l_scpy); - XPR(NT "target copies: %"Q"u (%"Q"u bytes)\n", - stream.n_tcpy, stream.l_tcpy); - XPR(NT "adds: %"Q"u (%"Q"u bytes)\n", stream.n_add, stream.l_add); - XPR(NT "runs: %"Q"u (%"Q"u bytes)\n", stream.n_run, stream.l_run); - } -#endif - - xd3_free_stream (& stream); - - if (option_verbose) - { - char tm[32]; - long end_time = get_millisecs_now (); - xoff_t nwrite = ofile != NULL ? ofile->nwrite : 0; - - XPR(NT "finished in %s; input %"Q"u output %"Q"u bytes (%0.2f%%)\n", - main_format_millis (end_time - start_time, tm), - ifile->nread, nwrite, 100.0 * nwrite / ifile->nread); - } - - return EXIT_SUCCESS; -} - -/* free memory before exit, reset single-use variables. */ -static void -main_cleanup (void) -{ - usize_t i; - - if (appheader_used != NULL && - appheader_used != option_appheader) - { - main_free (appheader_used); - appheader_used = NULL; - } - - main_free (main_bdata); - main_bdata = NULL; - main_bsize = 0; - -#if EXTERNAL_COMPRESSION - main_free (ext_tmpfile); - ext_tmpfile = NULL; -#endif - - for (i = 0; lru && i < lru_size; i += 1) - { - main_free (lru[i].blk); - } - - main_free (lru); - lru = NULL; - - lru_hits = 0; - lru_misses = 0; - lru_filled = 0; - - if (recode_stream != NULL) - { - xd3_free_stream (recode_stream); - main_free (recode_stream); - recode_stream = NULL; - } - - if (merge_stream != NULL) - { - xd3_free_stream (merge_stream); - main_free (merge_stream); - merge_stream = NULL; - } - - XD3_ASSERT (main_mallocs == 0); -} - -static void -setup_environment (int argc, - char **argv, - int *argc_out, - char ***argv_out, - char ***argv_free, - char **env_free) -{ - int n, i, i0; - char *p, *v = getenv("XDELTA"); - if (v == NULL) { - (*argc_out) = argc; - (*argv_out) = argv; - (*argv_free) = NULL; - (*env_free) = NULL; - return; - } - - (*env_free) = (char*) main_malloc(strlen(v) + 1); - strcpy(*env_free, v); - - /* Space needed for extra args, at least # of spaces */ - n = argc + 1; - for (p = *env_free; *p != 0; ) { - if (*p++ == ' ') { - n++; - } - } - - (*argv_free) = (char**) main_malloc(sizeof(char*) * (n + 1)); - (*argv_out) = (*argv_free); - (*argv_out)[0] = argv[0]; - (*argv_out)[n] = NULL; - - i = 1; - for (p = *env_free; *p != 0; ) { - (*argv_out)[i++] = p; - while (*p != ' ' && *p != 0) { - p++; - } - while (*p == ' ') { - *p++ = 0; - } - } - - for (i0 = 1; i0 < argc; i0++) { - (*argv_out)[i++] = argv[i0]; - } - - /* Counting spaces is an upper bound, argv stays NULL terminated. */ - (*argc_out) = i; - while (i <= n) { - (*argv_out)[i++] = NULL; - } -} - -int -#if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN -xd3_main_cmdline (int argc, char **argv) -#else -main (int argc, char **argv) -#endif -{ - static const char *flags = - "0123456789cdefhnqvDJNORTVs:m:B:C:E:F:I:L:O:M:P:W:A::S::"; - xd3_cmd cmd; - main_file ifile; - main_file ofile; - main_file sfile; - main_merge_list merge_order; - main_merge *merge; - int my_optind; - char *my_optarg; - char *my_optstr; - char *sfilename; - int env_argc; - char **env_argv; - char **free_argv; /* malloc() in setup_environment() */ - char *free_value; /* malloc() in setup_environment() */ - int ret; - -#ifdef _WIN32 - GetStartupInfo(&winStartupInfo); - setvbuf(stderr, NULL, _IONBF, 0); /* Do not buffer stderr */ -#endif - - main_file_init (& ifile); - main_file_init (& ofile); - main_file_init (& sfile); - main_merge_list_init (& merge_order); - - reset_defaults(); - - free_argv = NULL; - free_value = NULL; - setup_environment(argc, argv, &env_argc, &env_argv, - &free_argv, &free_value); - cmd = CMD_NONE; - sfilename = NULL; - my_optind = 1; - argv = env_argv; - argc = env_argc; - program_name = env_argv[0]; - extcomp_types[0].recomp_cmdname = program_name; - extcomp_types[0].decomp_cmdname = program_name; - - takearg: - my_optarg = NULL; - my_optstr = argv[my_optind]; - - /* This doesn't use getopt() because it makes trouble for -P & python which - * reenter main() and thus care about freeing all memory. I never had much - * trust for getopt anyway, it's too opaque. This implements a fairly - * standard non-long-option getopt with support for named operations (e.g., - * "xdelta3 [encode|decode|printhdr...] < in > out"). */ - if (my_optstr) - { - if (*my_optstr == '-') { my_optstr += 1; } - else if (cmd == CMD_NONE) { goto nonflag; } - else { my_optstr = NULL; } - } - while (my_optstr) - { - char *s; - my_optarg = NULL; - if ((ret = *my_optstr++) == 0) { my_optind += 1; goto takearg; } - - /* Option handling: first check for one ':' following the option in - * flags, then check for two. The syntax allows: - * - * 1. -Afoo defines optarg="foo" - * 2. -A foo defines optarg="foo" - * 3. -A "" defines optarg="" (allows empty-string) - * 4. -A [EOA or -moreargs] error (mandatory case) - * 5. -A [EOA -moreargs] defines optarg=NULL (optional case) - * 6. -A=foo defines optarg="foo" - * 7. -A= defines optarg="" (mandatory case) - * 8. -A= defines optarg=NULL (optional case) - * - * See tests in test_command_line_arguments(). - */ - s = strchr (flags, ret); - if (s && s[1] && s[1] == ':') - { - int eqcase = 0; - int option = s[2] && s[2] == ':'; - - /* Case 1, set optarg to the remaining characters. */ - my_optarg = my_optstr; - my_optstr = ""; - - /* Case 2-5 */ - if (*my_optarg == 0) - { - /* Condition 4-5 */ - int have_arg = (my_optind < (argc - 1) && - *argv[my_optind+1] != '-'); - - if (! have_arg) - { - if (! option) - { - /* Case 4 */ - XPR(NT "-%c: requires an argument\n", ret); - ret = EXIT_FAILURE; - goto cleanup; - } - /* Case 5. */ - my_optarg = NULL; - } - else - { - /* Case 2-3. */ - my_optarg = argv[++my_optind]; - } - } - /* Case 6-8. */ - else if (*my_optarg == '=') - { - /* Remove the = in all cases. */ - my_optarg += 1; - eqcase = 1; - - if (option && *my_optarg == 0) - { - /* Case 8. */ - my_optarg = NULL; - } - } - } - - switch (ret) - { - /* case: if no '-' was found, maybe check for a command name. */ - nonflag: - if (strcmp (my_optstr, "decode") == 0) { cmd = CMD_DECODE; } - else if (strcmp (my_optstr, "encode") == 0) - { -#if XD3_ENCODER - cmd = CMD_ENCODE; -#else - XPR(NT "encoder support not compiled\n"); - return EXIT_FAILURE; -#endif - } - else if (strcmp (my_optstr, "config") == 0) { cmd = CMD_CONFIG; } -#if REGRESSION_TEST - else if (strcmp (my_optstr, "test") == 0) { cmd = CMD_TEST; } -#endif -#if VCDIFF_TOOLS - else if (strcmp (my_optstr, "printhdr") == 0) { cmd = CMD_PRINTHDR; } - else if (strcmp (my_optstr, "printhdrs") == 0) - { cmd = CMD_PRINTHDRS; } - else if (strcmp (my_optstr, "printdelta") == 0) - { cmd = CMD_PRINTDELTA; } - else if (strcmp (my_optstr, "recode") == 0) { cmd = CMD_RECODE; } - else if (strcmp (my_optstr, "merge") == 0) { cmd = CMD_MERGE; } -#endif - - /* If no option was found and still no command, let the default - * command be encode. The remaining args are treated as - * filenames. */ - if (cmd == CMD_NONE) - { - cmd = CMD_DEFAULT; - my_optstr = NULL; - break; - } - else - { - /* But if we find a command name, continue the getopt loop. */ - my_optind += 1; - goto takearg; - } - - /* gzip-like options */ - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - option_level = ret - '0'; - break; - case 'f': option_force = 1; break; - case 'v': option_verbose += 1; option_quiet = 0; break; - case 'q': option_quiet = 1; option_verbose = 0; break; - case 'c': option_stdout = 1; break; - case 'd': - if (cmd == CMD_NONE) { cmd = CMD_DECODE; } - else { ret = main_help (); goto exit; } - break; - case 'e': -#if XD3_ENCODER - if (cmd == CMD_NONE) { cmd = CMD_ENCODE; } - else { ret = main_help (); goto exit; } - break; -#else - XPR(NT "encoder support not compiled\n"); - return EXIT_FAILURE; -#endif - - case 'n': option_use_checksum = 0; break; - case 'N': option_no_compress = 1; break; - case 'T': option_use_altcodetable = 1; break; - case 'C': option_smatch_config = my_optarg; break; - case 'J': option_no_output = 1; break; - case 'S': if (my_optarg == NULL) - { - option_use_secondary = 1; - option_secondary = "none"; - } - else - { - option_use_secondary = 1; - option_secondary = my_optarg; - } - break; - case 'A': if (my_optarg == NULL) { option_use_appheader = 0; } - else { option_appheader = (uint8_t*) my_optarg; } break; - case 'B': - if ((ret = main_atou (my_optarg, & option_srcwinsz, XD3_MINSRCWINSZ, - 0, 'B'))) - { - goto exit; - } - break; - case 'I': - if ((ret = main_atou (my_optarg, & option_iopt_size, 0, - 0, 'I'))) - { - goto exit; - } - break; - case 'P': - if ((ret = main_atou (my_optarg, & option_sprevsz, 0, - 0, 'P'))) - { - goto exit; - } - break; - case 'W': - if ((ret = main_atou (my_optarg, & option_winsize, XD3_ALLOCSIZE, - XD3_HARDMAXWINSIZE, 'W'))) - { - goto exit; - } - break; - case 'D': -#if EXTERNAL_COMPRESSION == 0 - if (option_verbose > 0) - { - XPR(NT "warning: -D option ignored, " - "external compression support was not compiled\n"); - } -#else - option_decompress_inputs = 0; -#endif - break; - case 'R': -#if EXTERNAL_COMPRESSION == 0 - if (option_verbose > 0) - { - XPR(NT "warning: -R option ignored, " - "external compression support was not compiled\n"); - } -#else - option_recompress_outputs = 0; -#endif - break; - case 's': - if (sfilename != NULL) - { - XPR(NT "specify only one source file\n"); - goto cleanup; - } - - sfilename = my_optarg; - break; - case 'm': - if ((merge = (main_merge*) - main_malloc (sizeof (main_merge))) == NULL) - { - goto cleanup; - } - main_merge_list_push_back (& merge_order, merge); - merge->filename = my_optarg; - break; - case 'V': - ret = main_version (); goto exit; - default: - ret = main_help (); goto exit; - } - } - - option_source_filename = sfilename; - - /* In case there were no arguments, set the default command. */ - if (cmd == CMD_NONE) { cmd = CMD_DEFAULT; } - - argc -= my_optind; - argv += my_optind; - - /* There may be up to two more arguments. */ - if (argc > 2) - { - XPR(NT "too many filenames: %s ...\n", argv[2]); - goto cleanup; - } - - ifile.flags = RD_FIRST; - sfile.flags = RD_FIRST; - sfile.filename = option_source_filename; - - /* The infile takes the next argument, if there is one. But if not, infile - * is set to stdin. */ - if (argc > 0) - { - ifile.filename = argv[0]; - - if ((ret = main_file_open (& ifile, ifile.filename, XO_READ))) - { - goto cleanup; - } - } - else - { - XSTDIN_XF (& ifile); - } - - /* The ofile takes the following argument, if there is one. But if not, it - * is left NULL until the application header is processed. It will be set - * in main_open_output. */ - if (argc > 1) - { - /* Check for conflicting arguments. */ - if (option_stdout && ! option_quiet) - { - XPR(NT "warning: -c option overrides output filename: %s\n", - argv[1]); - } - - if (! option_stdout) { ofile.filename = argv[1]; } - } - -#if VCDIFF_TOOLS - if (cmd == CMD_MERGE && - (ret = main_merge_arguments (&merge_order))) - { - goto cleanup; - } -#endif /* VCDIFF_TOOLS */ - - switch (cmd) - { - case CMD_PRINTHDR: - case CMD_PRINTHDRS: - case CMD_PRINTDELTA: -#if XD3_ENCODER - case CMD_ENCODE: - case CMD_RECODE: - case CMD_MERGE: -#endif - case CMD_DECODE: - ret = main_input (cmd, & ifile, & ofile, & sfile); - break; - -#if REGRESSION_TEST - case CMD_TEST: - main_config (); - ret = xd3_selftest (); - break; -#endif - - case CMD_CONFIG: - ret = main_config (); - break; - - default: - ret = main_help (); - break; - } - - if (0) - { - cleanup: - ret = EXIT_FAILURE; - exit: - (void)0; - } - -#if EXTERNAL_COMPRESSION - if (ext_tmpfile != NULL) - { - unlink (ext_tmpfile); - } -#endif - - main_file_cleanup (& ifile); - main_file_cleanup (& ofile); - main_file_cleanup (& sfile); - - while (! main_merge_list_empty (& merge_order)) - { - merge = main_merge_list_pop_front (& merge_order); - main_free (merge); - } - - main_free (free_argv); - main_free (free_value); - - main_cleanup (); - - fflush (stdout); - fflush (stderr); - return ret; -} - -static int -main_help (void) -{ - main_version(); - - /* Note: update wiki when command-line features change */ - DP(RINT "usage: xdelta3 [command/options] [input [output]]\n"); - DP(RINT "special command names:\n"); - DP(RINT " config prints xdelta3 configuration\n"); - DP(RINT " decode decompress the input\n"); - DP(RINT " encode compress the input%s\n", - XD3_ENCODER ? "" : " [Not compiled]"); -#if REGRESSION_TEST - DP(RINT " test run the builtin tests\n"); -#endif -#if VCDIFF_TOOLS - DP(RINT "special commands for VCDIFF inputs:\n"); - DP(RINT " printdelta print information about the entire delta\n"); - DP(RINT " printhdr print information about the first window\n"); - DP(RINT " printhdrs print information about all windows\n"); - DP(RINT " recode encode with new application/secondary settings\n"); - DP(RINT " merge merge VCDIFF inputs (see below)\n"); -#endif - DP(RINT "standard options:\n"); - DP(RINT " -0 .. -9 compression level\n"); - DP(RINT " -c use stdout\n"); - DP(RINT " -d decompress\n"); - DP(RINT " -e compress%s\n", - XD3_ENCODER ? "" : " [Not compiled]"); - DP(RINT " -f force overwrite\n"); - DP(RINT " -h show help\n"); - DP(RINT " -q be quiet\n"); - DP(RINT " -v be verbose (max 2)\n"); - DP(RINT " -V show version\n"); - - DP(RINT "memory options:\n"); - DP(RINT " -B bytes source window size\n"); - DP(RINT " -W bytes input window size\n"); - DP(RINT " -P size compression duplicates window\n"); - DP(RINT " -I size instruction buffer size (0 = unlimited)\n"); - - DP(RINT "compression options:\n"); - DP(RINT " -s source source file to copy from (if any)\n"); - DP(RINT " -S [djw|fgk] enable/disable secondary compression\n"); - DP(RINT " -N disable small string-matching compression\n"); - DP(RINT " -D disable external decompression (encode/decode)\n"); - DP(RINT " -R disable external recompression (decode)\n"); - DP(RINT " -n disable checksum (encode/decode)\n"); - DP(RINT " -C soft config (encode, undocumented)\n"); - DP(RINT " -A [apphead] disable/provide application header (encode)\n"); - DP(RINT " -J disable output (check/compute only)\n"); - DP(RINT " -T use alternate code table (test)\n"); - DP(RINT " -m arguments for \"merge\"\n"); - - DP(RINT "the XDELTA environment variable may contain extra args:\n"); - DP(RINT " XDELTA=\"-s source-x.y.tar.gz\" \\\n"); - DP(RINT " tar --use-compress-program=xdelta3 \\\n"); - DP(RINT " -cf target-x.z.tar.gz.vcdiff target-x.y\n"); - DP(RINT "the \"merge\" command combines VCDIFF inputs as follows:\n"); - DP(RINT " xdelta3 merge -m 1.vcdiff -m 2.vcdiff 3.vcdiff merged.vcdiff\n"); - return EXIT_FAILURE; -} diff --git a/xdelta3-merge.h b/xdelta3-merge.h deleted file mode 100644 index 2253a2c..0000000 --- a/xdelta3-merge.h +++ /dev/null @@ -1,579 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _XDELTA3_MERGE_H_ -#define _XDELTA3_MERGE_H_ - -int xd3_merge_inputs (xd3_stream *stream, - xd3_whole_state *source, - xd3_whole_state *input); - -static int -xd3_whole_state_init (xd3_stream *stream) -{ - XD3_ASSERT (stream->whole_target.adds == NULL); - XD3_ASSERT (stream->whole_target.inst == NULL); - XD3_ASSERT (stream->whole_target.wininfo == NULL); - XD3_ASSERT (stream->whole_target.length == 0); - - stream->whole_target.adds_alloc = XD3_ALLOCSIZE; - stream->whole_target.inst_alloc = XD3_ALLOCSIZE; - stream->whole_target.wininfo_alloc = XD3_ALLOCSIZE; - - if ((stream->whole_target.adds = (uint8_t*) - xd3_alloc (stream, stream->whole_target.adds_alloc, 1)) == NULL || - (stream->whole_target.inst = (xd3_winst*) - xd3_alloc (stream, stream->whole_target.inst_alloc, 1)) == NULL || - (stream->whole_target.wininfo = (xd3_wininfo*) - xd3_alloc (stream, stream->whole_target.wininfo_alloc, 1)) == NULL) - { - return ENOMEM; - } - return 0; -} - -static void -xd3_swap_whole_state (xd3_whole_state *a, - xd3_whole_state *b) -{ - xd3_whole_state tmp; - XD3_ASSERT (a->inst != NULL && a->adds != NULL); - XD3_ASSERT (b->inst != NULL && b->adds != NULL); - XD3_ASSERT (b->wininfo != NULL && b->wininfo != NULL); - memcpy (&tmp, a, sizeof (xd3_whole_state)); - memcpy (a, b, sizeof (xd3_whole_state)); - memcpy (b, &tmp, sizeof (xd3_whole_state)); -} - -static int -xd3_realloc_buffer (xd3_stream *stream, - usize_t current_units, - usize_t unit_size, - usize_t new_units, - usize_t *alloc_size, - void **alloc_ptr) -{ - usize_t needed; - usize_t new_alloc; - usize_t cur_size; - uint8_t *new_buf; - - needed = (current_units + new_units) * unit_size; - - if (needed <= *alloc_size) - { - return 0; - } - - cur_size = current_units * unit_size; - new_alloc = xd3_round_blksize (needed * 2, XD3_ALLOCSIZE); - - if ((new_buf = (uint8_t*) xd3_alloc (stream, new_alloc, 1)) == NULL) - { - return ENOMEM; - } - - if (cur_size != 0) - { - memcpy (new_buf, *alloc_ptr, cur_size); - } - - if (*alloc_ptr != NULL) - { - xd3_free (stream, *alloc_ptr); - } - - *alloc_size = new_alloc; - *alloc_ptr = new_buf; - - return 0; -} - -/* allocate one new output instruction */ -static int -xd3_whole_alloc_winst (xd3_stream *stream, - xd3_winst **winstp) -{ - int ret; - - if ((ret = xd3_realloc_buffer (stream, - stream->whole_target.instlen, - sizeof (xd3_winst), - 1, - & stream->whole_target.inst_alloc, - (void**) & stream->whole_target.inst))) - { - return ret; - } - - *winstp = &stream->whole_target.inst[stream->whole_target.instlen++]; - - return 0; -} - -static int -xd3_whole_alloc_adds (xd3_stream *stream, - usize_t count) -{ - return xd3_realloc_buffer (stream, - stream->whole_target.addslen, - 1, - count, - & stream->whole_target.adds_alloc, - (void**) & stream->whole_target.adds); -} - -static int -xd3_whole_alloc_wininfo (xd3_stream *stream, - xd3_wininfo **wininfop) -{ - int ret; - - if ((ret = xd3_realloc_buffer (stream, - stream->whole_target.wininfolen, - sizeof (xd3_wininfo), - 1, - & stream->whole_target.wininfo_alloc, - (void**) & stream->whole_target.wininfo))) - { - return ret; - } - - *wininfop = &stream->whole_target.wininfo[stream->whole_target.wininfolen++]; - - return 0; -} - -static int -xd3_whole_append_inst (xd3_stream *stream, - xd3_hinst *inst) -{ - int ret; - xd3_winst *winst; - - if ((ret = xd3_whole_alloc_winst (stream, &winst))) - { - return ret; - } - - winst->type = inst->type; - winst->mode = 0; - winst->size = inst->size; - winst->position = stream->whole_target.length; - stream->whole_target.length += inst->size; - - if (((inst->type == XD3_ADD) || (inst->type == XD3_RUN)) && - (ret = xd3_whole_alloc_adds (stream, - (inst->type == XD3_RUN ? 1 : inst->size)))) - { - return ret; - } - - switch (inst->type) - { - case XD3_RUN: - winst->addr = stream->whole_target.addslen; - stream->whole_target.adds[stream->whole_target.addslen++] = - *stream->data_sect.buf++; - break; - - case XD3_ADD: - winst->addr = stream->whole_target.addslen; - memcpy (stream->whole_target.adds + stream->whole_target.addslen, - stream->data_sect.buf, - inst->size); - stream->data_sect.buf += inst->size; - stream->whole_target.addslen += inst->size; - break; - - default: - if (inst->addr < stream->dec_cpylen) - { - winst->mode = SRCORTGT (stream->dec_win_ind); - winst->addr = stream->dec_cpyoff + inst->addr; - } - else - { - winst->addr = (stream->dec_winstart + - inst->addr - - stream->dec_cpylen); - } - break; - } - - return 0; -} - -int -xd3_whole_append_window (xd3_stream *stream) -{ - int ret; - xd3_wininfo *wininfo; - - if ((ret = xd3_whole_alloc_wininfo (stream, &wininfo))) { return ret; } - - wininfo->length = stream->dec_tgtlen; - wininfo->offset = stream->dec_winstart; - wininfo->adler32 = stream->dec_adler32; - - while (stream->inst_sect.buf < stream->inst_sect.buf_max) - { - if ((ret = xd3_decode_instruction (stream))) - { - return ret; - } - - if ((stream->dec_current1.type != XD3_NOOP) && - (ret = xd3_whole_append_inst (stream, - & stream->dec_current1))) - { - return ret; - } - - if ((stream->dec_current2.type != XD3_NOOP) && - (ret = xd3_whole_append_inst (stream, - & stream->dec_current2))) - { - return ret; - } - } - - return 0; -} - -/* xd3_merge_input_output applies *source to *stream, returns the - * result in stream. */ -int xd3_merge_input_output (xd3_stream *stream, - xd3_whole_state *source) -{ - int ret; - xd3_stream tmp_stream; - memset (& tmp_stream, 0, sizeof (tmp_stream)); - if ((ret = xd3_config_stream (& tmp_stream, NULL)) || - (ret = xd3_whole_state_init (& tmp_stream)) || - (ret = xd3_merge_inputs (& tmp_stream, - source, - & stream->whole_target))) - { - XPR(NT XD3_LIB_ERRMSG (&tmp_stream, ret)); - return ret; - } - - /* the output is in tmp_stream.whole_state, swap into input */ - xd3_swap_whole_state (& stream->whole_target, - & tmp_stream.whole_target); - /* total allocation counts are preserved */ - xd3_free_stream (& tmp_stream); - return 0; -} - -static int -xd3_merge_run (xd3_stream *stream, - xd3_whole_state *target, - xd3_winst *iinst) -{ - int ret; - xd3_winst *oinst; - - if ((ret = xd3_whole_alloc_winst (stream, &oinst)) || - (ret = xd3_whole_alloc_adds (stream, 1))) - { - return ret; - } - - oinst->type = iinst->type; - oinst->mode = iinst->mode; - oinst->size = iinst->size; - oinst->addr = stream->whole_target.addslen; - - XD3_ASSERT (stream->whole_target.length == iinst->position); - oinst->position = stream->whole_target.length; - stream->whole_target.length += iinst->size; - - stream->whole_target.adds[stream->whole_target.addslen++] = - target->adds[iinst->addr]; - - return 0; -} - -static int -xd3_merge_add (xd3_stream *stream, - xd3_whole_state *target, - xd3_winst *iinst) -{ - int ret; - xd3_winst *oinst; - - if ((ret = xd3_whole_alloc_winst (stream, &oinst)) || - (ret = xd3_whole_alloc_adds (stream, iinst->size))) - { - return ret; - } - - oinst->type = iinst->type; - oinst->mode = iinst->mode; - oinst->size = iinst->size; - oinst->addr = stream->whole_target.addslen; - - XD3_ASSERT (stream->whole_target.length == iinst->position); - oinst->position = stream->whole_target.length; - stream->whole_target.length += iinst->size; - - memcpy(stream->whole_target.adds + stream->whole_target.addslen, - target->adds + iinst->addr, - iinst->size); - - stream->whole_target.addslen += iinst->size; - - return 0; -} - -static int -xd3_merge_target_copy (xd3_stream *stream, - xd3_winst *iinst) -{ - int ret; - xd3_winst *oinst; - - if ((ret = xd3_whole_alloc_winst (stream, &oinst))) - { - return ret; - } - - XD3_ASSERT (stream->whole_target.length == iinst->position); - - memcpy (oinst, iinst, sizeof (*oinst)); - return 0; -} - -static int -xd3_merge_find_position (xd3_stream *stream, - xd3_whole_state *source, - xoff_t address, - usize_t *inst_num) -{ - usize_t low; - usize_t high; - - if (address >= source->length) - { - stream->msg = "Invalid copy offset in merge"; - return XD3_INVALID_INPUT; - } - - low = 0; - high = source->instlen; - - while (low != high) - { - xoff_t mid_lpos; - xoff_t mid_hpos; - usize_t mid = low + (high - low) / 2; - mid_lpos = source->inst[mid].position; - - if (address < mid_lpos) - { - high = mid; - continue; - } - - mid_hpos = mid_lpos + source->inst[mid].size; - - if (address >= mid_hpos) - { - low = mid + 1; - continue; - } - - *inst_num = mid; - return 0; - } - - stream->msg = "Internal error in merge"; - return XD3_INTERNAL; -} - -static int -xd3_merge_source_copy (xd3_stream *stream, - xd3_whole_state *source, - const xd3_winst *iinst_orig) -{ - int ret; - xd3_winst iinst; - usize_t sinst_num; - - memcpy (& iinst, iinst_orig, sizeof (iinst)); - - XD3_ASSERT (iinst.mode == VCD_SOURCE); - - if ((ret = xd3_merge_find_position (stream, source, - iinst.addr, &sinst_num))) - { - return ret; - } - - while (iinst.size > 0) - { - xd3_winst *sinst; - xd3_winst *minst; - usize_t sinst_offset; - usize_t sinst_left; - usize_t this_take; - - XD3_ASSERT (sinst_num < source->instlen); - - sinst = &source->inst[sinst_num]; - - XD3_ASSERT (iinst.addr >= sinst->position); - - sinst_offset = iinst.addr - sinst->position; - - XD3_ASSERT (sinst->size > sinst_offset); - - sinst_left = sinst->size - sinst_offset; - this_take = min (iinst.size, sinst_left); - - XD3_ASSERT (this_take > 0); - - if ((ret = xd3_whole_alloc_winst (stream, &minst))) - { - return ret; - } - - minst->size = this_take; - minst->type = sinst->type; - minst->position = iinst.position; - minst->mode = 0; - - switch (sinst->type) - { - case XD3_RUN: - if ((ret = xd3_whole_alloc_adds (stream, 1))) - { - return ret; - } - - minst->addr = stream->whole_target.addslen; - stream->whole_target.adds[stream->whole_target.addslen++] = - source->adds[sinst->addr]; - break; - case XD3_ADD: - if ((ret = xd3_whole_alloc_adds (stream, this_take))) - { - return ret; - } - - minst->addr = stream->whole_target.addslen; - memcpy(stream->whole_target.adds + stream->whole_target.addslen, - source->adds + sinst->addr + sinst_offset, - this_take); - stream->whole_target.addslen += this_take; - break; - default: - if (sinst->mode != 0) - { - minst->mode = sinst->mode; - minst->addr = sinst->addr + sinst_offset; - } - else - { - // TODO: this is slow because of the recursion, which - // could reach a depth equal to the number of target - // copies, and this is compression-inefficient because - // it can produce duplicate adds. - xd3_winst tinst; - tinst.type = XD3_CPY; - tinst.mode = iinst.mode; - tinst.addr = sinst->addr + sinst_offset; - tinst.size = this_take; - tinst.position = iinst.position; - - // The instruction allocated in this frame will not be used. - stream->whole_target.instlen -= 1; - - if ((ret = xd3_merge_source_copy (stream, source, &tinst))) - { - return ret; - } - } - break; - } - - iinst.position += this_take; - iinst.addr += this_take; - iinst.size -= this_take; - sinst_num += 1; - } - - return 0; -} - -/* xd3_merge_inputs() applies *input to *source, returns its result in - * stream. */ -int xd3_merge_inputs (xd3_stream *stream, - xd3_whole_state *source, - xd3_whole_state *input) -{ - int ret = 0; - usize_t i; - size_t input_i; - - for (i = 0; i < input->wininfolen; ++i) { - xd3_wininfo *copyinfo; - - if ((ret = xd3_whole_alloc_wininfo (stream, ©info))) { return ret; } - - *copyinfo = input->wininfo[i]; - } - - /* iterate over each instruction. */ - for (input_i = 0; ret == 0 && input_i < input->instlen; ++input_i) - { - xd3_winst *iinst = &input->inst[input_i]; - - switch (iinst->type) - { - case XD3_RUN: - ret = xd3_merge_run (stream, input, iinst); - break; - case XD3_ADD: - ret = xd3_merge_add (stream, input, iinst); - break; - default: - /* TODO: VCD_TARGET support is completely untested all - * throughout. */ - if (iinst->mode == 0 || iinst->mode == VCD_TARGET) - { - ret = xd3_merge_target_copy (stream, iinst); - } - else - { - ret = xd3_merge_source_copy (stream, source, iinst); - } - - /* The whole_target.length is not updated in the xd3_merge*copy - * routine because of recursion in xd3_merge_source_copy. */ - stream->whole_target.length += iinst->size; - break; - } - } - - return ret; -} - -#endif diff --git a/xdelta3-python.h b/xdelta3-python.h deleted file mode 100644 index 4805b17..0000000 --- a/xdelta3-python.h +++ /dev/null @@ -1,88 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include "Python.h" - -static PyObject *pyxd3_error; - -/* spam: xdelta3.main([string,list,...]) */ -PyObject *xdelta3_main_cmdline (PyObject *self, PyObject *args) -{ - int ret, i, nargs; - char **argv = NULL; - int argc = 0; - PyObject *result = NULL; - PyObject *o; - - if (! PyArg_ParseTuple (args, "O", &o) - || ! PyList_Check (o)) - { - goto cleanup; - } - - argc = PyList_Size (o); - nargs = argc + 2; - - if (! (argv = malloc (sizeof(argv[0]) * nargs))) - { - PyErr_NoMemory (); - goto cleanup; - } - memset (argv, 0, sizeof(argv[0]) * nargs); - - for (i = 1; i < nargs-1; i += 1) - { - char *ps; - PyObject *s; - if ((s = PyList_GetItem (o, i-1)) == NULL) { goto cleanup; } - ps = PyString_AsString (s); - /* TODO: ps is NULL if s is not a string, crashes the interpreter */ - argv[i] = ps; - } - - ret = xd3_main_cmdline (argc+1, argv); - - if (ret == 0) - { - result = Py_BuildValue ("i", ret); - } - else - { - PyErr_SetString (pyxd3_error, "failed :("); - } - cleanup: - if (argv) - { - free (argv); - } - return result; -} - -static PyMethodDef xdelta3_methods[] = { - { "main", xdelta3_main_cmdline, METH_VARARGS, "xdelta3 main()" }, - { NULL, NULL } -}; - -DL_EXPORT(void) initxdelta3main (void) -{ - PyObject *m, *d; - m = Py_InitModule ("xdelta3main", xdelta3_methods); - d = PyModule_GetDict (m); - pyxd3_error = PyErr_NewException ("xdelta3main.error", NULL, NULL); - PyDict_SetItemString (d, "error", pyxd3_error); -} diff --git a/xdelta3-regtest.py b/xdelta3-regtest.py deleted file mode 100755 index f9a11bd..0000000 --- a/xdelta3-regtest.py +++ /dev/null @@ -1,1222 +0,0 @@ -#!/usr/bin/python2.5 -# xdelta 3 - delta compression tools and library -# Copyright (C) 2003, 2006, 2007, 2008. Joshua P. MacDonald -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -# TODO: test 1.5 vs. greedy - -import os, sys, math, re, time, types, array, random -import xdelta3 - -#RCSDIR = '/mnt/polaroid/Polaroid/orbit_linux/home/jmacd/PRCS' -#RCSDIR = '/tmp/PRCS_read_copy' -#SAMPLEDIR = "/tmp/WESNOTH_tmp/diff" - -#RCSDIR = 'G:/jmacd/PRCS_copy' -#SAMPLEDIR = "C:/sample_data/Wesnoth/tar" - -#RCSDIR = '/Users/jmacd/src/ftp.kernel.org/pub/scm/linux/kernel/bkcvs/linux-2.4/net/x25' -RCSDIR = '/Users/jmacd/src/ftp.kernel.org' - -# -MIN_SIZE = 0 - -TIME_TOO_SHORT = 0.050 - -SKIP_TRIALS = 2 -MIN_TRIALS = 3 -MAX_TRIALS = 15 - -# 10 = fast 1.5 = slow -MIN_STDDEV_PCT = 1.5 - -# How many results per round -MAX_RESULTS = 500 -TEST_ROUNDS = 500 -KEEP_P = (0.5) - -# For RCS testing, what percent to select -FILE_P = (0.50) - -# For run-speed tests -MIN_RUN = 1000 * 1000 * 1 -MAX_RUN = 1000 * 1000 * 10 - -# Testwide defaults -ALL_ARGS = [ - '-vv' - ] - -# The first 7 args go to -C -SOFT_CONFIG_CNT = 7 - -CONFIG_ORDER = [ 'large_look', - 'large_step', - 'small_look', - 'small_chain', - 'small_lchain', - 'max_lazy', - 'long_enough', - - # > SOFT_CONFIG_CNT - 'nocompress', - 'winsize', - 'srcwinsize', - 'sprevsz', - 'iopt', - 'djw', - 'altcode', - ] - -CONFIG_ARGMAP = { - 'winsize' : '-W', - 'srcwinsize' : '-B', - 'sprevsz' : '-P', - 'iopt' : '-I', - 'nocompress' : '-N', - 'djw' : '-Sdjw', - 'altcode' : '-T', - } - -def INPUT_SPEC(rand): - return { - - # Time/space costs: - - # -C 1,2,3,4,5,6,7 - 'large_look' : lambda d: rand.choice([9, 10, 11, 12]), - 'large_step' : lambda d: rand.choice([25, 26, 27, 28, 29, 30]), - 'small_look' : lambda d: rand.choice([4]), - 'small_chain' : lambda d: rand.choice([1]), - 'small_lchain' : lambda d: rand.choice([1]), - 'max_lazy' : lambda d: rand.choice([4, 5, 6, 7, 8, 9, 10 ]), - - # Note: long_enough only refers to small matching and has no effect if - # small_chain == 1. - 'long_enough' : lambda d: rand.choice([4]), - - # -N - 'nocompress' : lambda d: rand.choice(['false']), - - # -T - 'altcode' : lambda d: rand.choice(['false']), - - # -S djw - 'djw' : lambda d: rand.choice(['false']), - - # Memory costs: - - # -W - 'winsize' : lambda d: 8 * (1<<20), - - # -B - 'srcwinsize' : lambda d: 64 * (1<<20), - - # -I 0 is unlimited - 'iopt' : lambda d: 0, - - # -P only powers of two - 'sprevsz' : lambda d: rand.choice([x * (1<<16) for x in [4]]), - } -#end - -# -TMPDIR = '/tmp/xd3regtest.%d' % os.getpid() - -RUNFILE = os.path.join(TMPDIR, 'run') -DFILE = os.path.join(TMPDIR, 'output') -RFILE = os.path.join(TMPDIR, 'recon') - -HEAD_STATE = 0 -BAR_STATE = 1 -REV_STATE = 2 -DATE_STATE = 3 - -# -IGNORE_FILENAME = re.compile('.*\\.(gif|jpg).*') - -# rcs output -RE_TOTREV = re.compile('total revisions: (\\d+)') -RE_BAR = re.compile('----------------------------') -RE_REV = re.compile('revision (.+)') -RE_DATE = re.compile('date: ([^;]+);.*') -# xdelta output -RE_HDRSZ = re.compile('VCDIFF header size: +(\\d+)') -RE_EXTCOMP = re.compile('XDELTA ext comp.*') - -def c2str(c): - return ' '.join(['%s' % x for x in c]) -#end - -def SumList(l): - return reduce(lambda x,y: x+y, l) -#end - -# returns (total, mean, stddev, q2 (median), -# (q3-q1)/2 ("semi-interquartile range"), max-min (spread)) -class StatList: - def __init__(self,l,desc): - cnt = len(l) - assert(cnt > 1) - l.sort() - self.cnt = cnt - self.l = l - self.total = SumList(l) - self.mean = self.total / float(self.cnt) - self.s = math.sqrt(SumList([(x-self.mean) * (x - self.mean) for x in l]) / float(self.cnt-1)) - self.q0 = l[0] - self.q1 = l[int(self.cnt/4.0+0.5)] - self.q2 = l[int(self.cnt/2.0+0.5)] - self.q3 = l[min(self.cnt-1,int((3.0*self.cnt)/4.0+0.5))] - self.q4 = l[self.cnt-1]+1 - self.siqr = (self.q3-self.q1)/2.0; - self.spread = (self.q4-self.q0) - self.str = '%s %d; mean %d; sdev %d; q2 %d; .5(q3-q1) %.1f; spread %d' % \ - (desc, self.total, self.mean, self.s, self.q2, self.siqr, self.spread) - #end -#end - -def RunCommand(args, ok = [0]): - #print 'run command %s' % (' '.join(args)) - p = os.spawnvp(os.P_WAIT, args[0], args) - if p not in ok: - raise CommandError(args, 'exited %d' % p) - #end -#end - -def RunCommandIO(args,infn,outfn): - p = os.fork() - if p == 0: - os.dup2(os.open(infn,os.O_RDONLY),0) - os.dup2(os.open(outfn,os.O_CREAT|os.O_TRUNC|os.O_WRONLY),1) - os.execvp(args[0], args) - else: - s = os.waitpid(p,0) - o = os.WEXITSTATUS(s[1]) - if not os.WIFEXITED(s[1]) or o != 0: - raise CommandError(args, 'exited %d' % o) - #end - #end -#end - -class TimedTest: - def __init__(self, target, source, runnable, - skip_trials = SKIP_TRIALS, - min_trials = MIN_TRIALS, - max_trials = MAX_TRIALS, - min_stddev_pct = MIN_STDDEV_PCT): - self.target = target - self.source = source - self.runnable = runnable - - self.skip_trials = skip_trials - self.min_trials = min(min_trials, max_trials) - self.max_trials = max_trials - self.min_stddev_pct = min_stddev_pct - - self.encode_time = self.DoTest(DFILE, - lambda x: x.Encode(self.target, self.source, DFILE)) - self.encode_size = runnable.EncodeSize(DFILE) - - self.decode_time = self.DoTest(RFILE, - lambda x: x.Decode(DFILE, self.source, RFILE), - ) - - # verify - runnable.Verify(self.target, RFILE) - #end - - def DoTest(self, fname, func): - trials = 0 - measured = [] - - while 1: - try: - os.remove(fname) - except OSError: - pass - - start_time = time.time() - start_clock = time.clock() - - func(self.runnable) - - total_clock = (time.clock() - start_clock) - total_time = (time.time() - start_time) - - elap_time = max(total_time, 0.0000001) - elap_clock = max(total_clock, 0.0000001) - - trials = trials + 1 - - # skip some of the first trials - if trials > self.skip_trials: - measured.append((elap_clock, elap_time)) - #print 'measurement total: %.1f ms' % (total_time * 1000.0) - - # at least so many - if trials < (self.skip_trials + self.min_trials): - #print 'continue: need more trials: %d' % trials - continue - - # compute %variance - done = 0 - if self.skip_trials + self.min_trials <= 2: - measured = measured + measured; - done = 1 - #end - - time_stat = StatList([x[1] for x in measured], 'elap time') - sp = float(time_stat.s) / float(time_stat.mean) - - # what if MAX_TRIALS is exceeded? - too_many = (trials - self.skip_trials) >= self.max_trials - good = (100.0 * sp) < self.min_stddev_pct - if done or too_many or good: - trials = trials - self.skip_trials - if not done and not good: - #print 'too many trials: %d' % trials - pass - #clock = StatList([x[0] for x in measured], 'elap clock') - return time_stat - #end - #end - #end -#end - -def Decimals(start, end): - l = [] - step = start - while 1: - r = range(step, step * 10, step) - l = l + r - if step * 10 >= end: - l.append(step * 10) - break - step = step * 10 - return l -#end - -# This tests the raw speed of 0-byte inputs -def RunSpeedTest(): - for L in Decimals(MIN_RUN, MAX_RUN): - SetFileSize(RUNFILE, L) - - trx = TimedTest(RUNFILE, None, Xdelta3Runner(['-W', str(1<<20)])) - ReportSpeed(L, trx, '1MB ') - - trx = TimedTest(RUNFILE, None, Xdelta3Runner(['-W', str(1<<19)])) - ReportSpeed(L, trx, '512k') - - trx = TimedTest(RUNFILE, None, Xdelta3Runner(['-W', str(1<<18)])) - ReportSpeed(L, trx, '256k') - - trm = TimedTest(RUNFILE, None, Xdelta3Mod1(RUNFILE)) - ReportSpeed(L, trm, 'swig') - - trg = TimedTest(RUNFILE, None, GzipRun1()) - ReportSpeed(L,trg,'gzip') - #end -#end - -def SetFileSize(F,L): - fd = os.open(F, os.O_CREAT | os.O_WRONLY) - os.ftruncate(fd,L) - assert os.fstat(fd).st_size == L - os.close(fd) -#end - -def ReportSpeed(L,tr,desc): - print '%s run length %u: size %u: time %.3f ms: decode %.3f ms' % \ - (desc, L, - tr.encode_size, - tr.encode_time.mean * 1000.0, - tr.decode_time.mean * 1000.0) -#end - -class Xdelta3RunClass: - def __init__(self, extra): - self.extra = extra - #end - - def __str__(self): - return ' '.join(self.extra) - #end - - def New(self): - return Xdelta3Runner(self.extra) - #end -#end - -class Xdelta3Runner: - def __init__(self, extra): - self.extra = extra - #end - - def Encode(self, target, source, output): - args = (ALL_ARGS + - self.extra + - ['-e']) - if source: - args.append('-s') - args.append(source) - #end - args = args + [target, output] - self.Main(args) - #end - - def Decode(self, input, source, output): - args = (ALL_ARGS + - ['-d']) - if source: - args.append('-s') - args.append(source) - #end - args = args + [input, output] - self.Main(args) - #end - - def Verify(self, target, recon): - RunCommand(('cmp', target, recon)) - #end - - def EncodeSize(self, output): - return os.stat(output).st_size - #end - - def Main(self, args): - try: - #print 'Run %s' % (' '.join(args)) - xdelta3.xd3_main_cmdline(args) - except Exception, e: - raise CommandError(args, "xdelta3.main exception: %s" % e) - #end - #end -#end - -class Xdelta3Mod1: - def __init__(self, file): - self.target_data = open(file, 'r').read() - #end - - def Encode(self, ignore1, ignore2, ignore3): - r1, encoded = xdelta3.xd3_encode_memory(self.target_data, None, 1000000, 1<<10) - if r1 != 0: - raise CommandError('memory', 'encode failed: %s' % r1) - #end - self.encoded = encoded - #end - - def Decode(self, ignore1, ignore2, ignore3): - r2, data1 = xdelta3.xd3_decode_memory(self.encoded, None, len(self.target_data)) - if r2 != 0: - raise CommandError('memory', 'decode failed: %s' % r1) - #end - self.decoded = data1 - #end - - def Verify(self, ignore1, ignore2): - if self.target_data != self.decoded: - raise CommandError('memory', 'bad decode') - #end - #end - - def EncodeSize(self, ignore1): - return len(self.encoded) - #end -#end - -class GzipRun1: - def Encode(self, target, source, output): - assert source == None - RunCommandIO(['gzip', '-cf'], target, output) - #end - - def Decode(self, input, source, output): - assert source == None - RunCommandIO(['gzip', '-dcf'], input, output) - #end - - def Verify(self, target, recon): - RunCommand(('cmp', target, recon)) - #end - - def EncodeSize(self, output): - return os.stat(output).st_size - #end -#end - -class Xdelta1RunClass: - def __str__(self): - return 'xdelta1' - #end - - def New(self): - return Xdelta1Runner() - #end -#end - -class Xdelta1Runner: - def Encode(self, target, source, output): - assert source != None - args = ['xdelta1', 'delta', '-q', source, target, output] - RunCommand(args, [0, 1]) - #end - - def Decode(self, input, source, output): - assert source != None - args = ['xdelta1', 'patch', '-q', input, source, output] - # Note: for dumb historical reasons, xdelta1 returns 1 or 0 - RunCommand(args) - #end - - def Verify(self, target, recon): - RunCommand(('cmp', target, recon)) - #end - - def EncodeSize(self, output): - return os.stat(output).st_size - #end -#end - -# exceptions -class SkipRcsException: - def __init__(self,reason): - self.reason = reason - #end -#end - -class NotEnoughVersions: - def __init__(self): - pass - #end -#end - -class CommandError: - def __init__(self,cmd,str): - if type(cmd) is types.TupleType or \ - type(cmd) is types.ListType: - cmd = reduce(lambda x,y: '%s %s' % (x,y),cmd) - #end - print 'command was: ',cmd - print 'command failed: ',str - print 'have fun debugging' - #end -#end - -class RcsVersion: - def __init__(self,vstr): - self.vstr = vstr - #end - def __cmp__(self,other): - return cmp(self.date, other.date) - #end - def __str__(self): - return str(self.vstr) - #end -#end - -class RcsFile: - - def __init__(self, fname): - self.fname = fname - self.versions = [] - self.state = HEAD_STATE - #end - - def SetTotRev(self,s): - self.totrev = int(s) - #end - - def Rev(self,s): - self.rev = RcsVersion(s) - if len(self.versions) >= self.totrev: - raise SkipRcsException('too many versions (in log messages)') - #end - self.versions.append(self.rev) - #end - - def Date(self,s): - self.rev.date = s - #end - - def Match(self, line, state, rx, gp, newstate, f): - if state == self.state: - m = rx.match(line) - if m: - if f: - f(m.group(gp)) - #end - self.state = newstate - return 1 - #end - #end - return None - #end - - def Sum1Rlog(self): - f = os.popen('rlog '+self.fname, "r") - l = f.readline() - while l: - if self.Match(l, HEAD_STATE, RE_TOTREV, 1, BAR_STATE, self.SetTotRev): - pass - elif self.Match(l, BAR_STATE, RE_BAR, 1, REV_STATE, None): - pass - elif self.Match(l, REV_STATE, RE_REV, 1, DATE_STATE, self.Rev): - pass - elif self.Match(l, DATE_STATE, RE_DATE, 1, BAR_STATE, self.Date): - pass - #end - l = f.readline() - #end - c = f.close() - if c != None: - raise c - #end - #end - - def Sum1(self): - st = os.stat(self.fname) - self.rcssize = st.st_size - self.Sum1Rlog() - if self.totrev != len(self.versions): - raise SkipRcsException('wrong version count') - #end - self.versions.sort() - #end - - def Checkout(self,n): - v = self.versions[n] - out = open(self.Verf(n), "w") - cmd = 'co -ko -p%s %s' % (v.vstr, self.fname) - total = 0 - (inf, - stream, - err) = os.popen3(cmd, "r") - inf.close() - buf = stream.read() - while buf: - total = total + len(buf) - out.write(buf) - buf = stream.read() - #end - v.vsize = total - estr = '' - buf = err.read() - while buf: - estr = estr + buf - buf = err.read() - #end - if stream.close(): - raise CommandError(cmd, 'checkout failed: %s\n%s\n%s' % (v.vstr, self.fname, estr)) - #end - out.close() - err.close() - #end - - def Vdate(self,n): - return self.versions[n].date - #end - - def Vstr(self,n): - return self.versions[n].vstr - #end - - def Verf(self,n): - return os.path.join(TMPDIR, 'input.%d' % n) - #end - - def FilePairsByDate(self, runclass): - if self.totrev < 2: - raise NotEnoughVersions() - #end - self.Checkout(0) - ntrials = [] - if self.totrev < 2: - return vtrials - #end - for v in range(0,self.totrev-1): - if v > 1: - os.remove(self.Verf(v-1)) - #end - self.Checkout(v+1) - if os.stat(self.Verf(v)).st_size < MIN_SIZE or \ - os.stat(self.Verf(v+1)).st_size < MIN_SIZE: - continue - #end - - result = TimedTest(self.Verf(v+1), - self.Verf(v), - runclass.New()) - - target_size = os.stat(self.Verf(v+1)).st_size - - ntrials.append(result) - #end - - os.remove(self.Verf(self.totrev-1)) - os.remove(self.Verf(self.totrev-2)) - return ntrials - #end - - def AppendVersion(self, f, n): - self.Checkout(n) - rf = open(self.Verf(n), "r") - data = rf.read() - f.write(data) - rf.close() - return len(data) - #end - -class RcsFinder: - def __init__(self): - self.subdirs = [] - self.rcsfiles = [] - self.others = [] - self.skipped = [] - self.biground = 0 - #end - - def Scan1(self,dir): - dents = os.listdir(dir) - subdirs = [] - rcsfiles = [] - others = [] - for dent in dents: - full = os.path.join(dir, dent) - if os.path.isdir(full): - subdirs.append(full) - elif dent[len(dent)-2:] == ",v": - rcsfiles.append(RcsFile(full)) - else: - others.append(full) - #end - #end - self.subdirs = self.subdirs + subdirs - self.rcsfiles = self.rcsfiles + rcsfiles - self.others = self.others + others - return subdirs - #end - - def Crawl(self, dir): - subdirs = [dir] - while subdirs: - s1 = self.Scan1(subdirs[0]) - subdirs = subdirs[1:] + s1 - #end - #end - - def Summarize(self): - good = [] - for rf in self.rcsfiles: - try: - rf.Sum1() - if rf.totrev < 2: - raise SkipRcsException('too few versions (< 2)') - #end - except SkipRcsException, e: - #print 'skipping file %s: %s' % (rf.fname, e.reason) - self.skipped.append(rf) - else: - good.append(rf) - #end - self.rcsfiles = good - #end - - def AllPairsByDate(self, runclass): - results = [] - good = [] - for rf in self.rcsfiles: - try: - results = results + rf.FilePairsByDate(runclass) - except SkipRcsException: - print 'file %s has compressed versions: skipping' % (rf.fname) - except NotEnoughVersions: - print 'testing %s on %s: not enough versions' % (runclass, rf.fname) - else: - good.append(rf) - #end - self.rcsfiles = good - self.ReportPairs(runclass, results) - return results - #end - - def ReportPairs(self, name, results): - encode_time = 0 - decode_time = 0 - encode_size = 0 - for r in results: - encode_time += r.encode_time.mean - decode_time += r.decode_time.mean - encode_size += r.encode_size - #end - print '%s rcs: encode %.2f s: decode %.2f s: size %d' % \ - (name, encode_time, decode_time, encode_size) - #end - - def MakeBigFiles(self, rand): - f1 = open(TMPDIR + "/big.1", "w") - f2 = open(TMPDIR + "/big.2", "w") - population = [] - for file in self.rcsfiles: - if len(file.versions) < 2: - continue - population.append(file) - #end - f1sz = 0 - f2sz = 0 - fcount = int(len(population) * FILE_P) - assert fcount > 0 - for file in rand.sample(population, fcount): - m = IGNORE_FILENAME.match(file.fname) - if m != None: - continue - #end - r1, r2 = rand.sample(xrange(0, len(file.versions)), 2) - f1sz += file.AppendVersion(f1, r1) - f2sz += file.AppendVersion(f2, r2) - #m.update('%s,%s,%s ' % (file.fname[len(RCSDIR):], file.Vstr(r1), file.Vstr(r2))) - #end - testkey = 'rcs%d' % self.biground - self.biground = self.biground + 1 - - print '%s; source %u bytes; target %u bytes' % (testkey, f1sz, f2sz) - f1.close() - f2.close() - return (TMPDIR + "/big.1", - TMPDIR + "/big.2", - testkey) - #end - - def Generator(self): - return lambda rand: self.MakeBigFiles(rand) - #end -#end - -# find a set of RCS files for testing -def GetTestRcsFiles(): - rcsf = RcsFinder() - rcsf.Crawl(RCSDIR) - if len(rcsf.rcsfiles) == 0: - raise CommandError('', 'no RCS files') - #end - rcsf.Summarize() - print "rcsfiles: rcsfiles %d; subdirs %d; others %d; skipped %d" % (len(rcsf.rcsfiles), - len(rcsf.subdirs), - len(rcsf.others), - len(rcsf.skipped)) - print StatList([x.rcssize for x in rcsf.rcsfiles], "rcssize").str - print StatList([x.totrev for x in rcsf.rcsfiles], "totrev").str - return rcsf -#end - -class SampleDataTest: - def __init__(self, dirs): - self.pairs = [] - while dirs: - d = dirs[0] - dirs = dirs[1:] - l = os.listdir(d) - files = [] - for e in l: - p = os.path.join(d, e) - if os.path.isdir(p): - dirs.append(p) - else: - files.append(p) - #end - #end - if len(files) > 1: - files.sort() - for x in xrange(len(files) - 1): - self.pairs.append((files[x], files[x+1], - '%s-%s' % (files[x], files[x+1]))) - #end - #end - #end - #end - - def Generator(self): - return lambda rand: rand.choice(self.pairs) - #end -#end - -# configs are represented as a list of values, -# program takes a list of strings: -def ConfigToArgs(config): - args = [ '-C', - ','.join([str(x) for x in config[0:SOFT_CONFIG_CNT]])] - for i in range(SOFT_CONFIG_CNT, len(CONFIG_ORDER)): - key = CONFIG_ARGMAP[CONFIG_ORDER[i]] - val = config[i] - if val == 'true' or val == 'false': - if val == 'true': - args.append('%s' % key) - #end - else: - args.append('%s=%s' % (key, val)) - #end - #end - return args -#end - -# -class RandomTest: - def __init__(self, tnum, tinput, config, syntuple = None): - self.mytinput = tinput[2] - self.myconfig = config - self.tnum = tnum - - if syntuple != None: - self.runtime = syntuple[0] - self.compsize = syntuple[1] - self.decodetime = None - else: - args = ConfigToArgs(config) - result = TimedTest(tinput[1], tinput[0], Xdelta3Runner(args)) - - self.runtime = result.encode_time.mean - self.compsize = result.encode_size - self.decodetime = result.decode_time.mean - #end - - self.score = None - self.time_pos = None - self.size_pos = None - self.score_pos = None - #end - - def __str__(self): - decodestr = ' %.6f' % self.decodetime - return 'time %.6f%s size %d%s << %s >>%s' % ( - self.time(), ((self.time_pos != None) and (" (%s)" % self.time_pos) or ""), - self.size(), ((self.size_pos != None) and (" (%s)" % self.size_pos) or ""), - c2str(self.config()), - decodestr) - #end - - def time(self): - return self.runtime - #end - - def size(self): - return self.compsize - #end - - def config(self): - return self.myconfig - #end - - def score(self): - return self.score - #end - - def tinput(self): - return self.mytinput - #end -#end - -def PosInAlist(l, e): - for i in range(0, len(l)): - if l[i][1] == e: - return i; - #end - #end - return -1 -#end - -# Generates a set of num_results test configurations, given the list of -# retest-configs. -def RandomTestConfigs(rand, input_configs, num_results): - - outputs = input_configs[:] - have_set = dict([(c,c) for c in input_configs]) - - # Compute a random configuration - def RandomConfig(): - config = [] - cmap = {} - for key in CONFIG_ORDER: - val = cmap[key] = (INPUT_SPEC(rand)[key])(cmap) - config.append(val) - #end - return tuple(config) - #end - - while len(outputs) < num_results: - newc = None - for i in xrange(100): - c = RandomConfig() - if have_set.has_key(c): - continue - #end - have_set[c] = c - newc = c - break - if newc is None: - print 'stopped looking for configs at %d' % len(outputs) - break - #end - outputs.append(c) - #end - outputs.sort() - return outputs -#end - -def RunTestLoop(rand, generator, rounds): - configs = [] - for rnum in xrange(rounds): - configs = RandomTestConfigs(rand, configs, MAX_RESULTS) - tinput = generator(rand) - tests = [] - for x in xrange(len(configs)): - t = RandomTest(x, tinput, configs[x]) - print 'Round %d test %d: %s' % (rnum, x, t) - tests.append(t) - #end - results = ScoreTests(tests) - - for r in results: - c = r.config() - if not test_all_config_results.has_key(c): - test_all_config_results[c] = [r] - else: - test_all_config_results[c].append(r) - #end - #end - - GraphResults('expt%d' % rnum, results) - GraphSummary('sum%d' % rnum, results) - - # re-test some fraction - configs = [r.config() for r in results[0:int(MAX_RESULTS * KEEP_P)]] - #end -#end - -# TODO: cleanup -test_all_config_results = {} - -def ScoreTests(results): - scored = [] - timed = [] - sized = [] - - t_min = float(min([test.time() for test in results])) - #t_max = float(max([test.time() for test in results])) - s_min = float(min([test.size() for test in results])) - #s_max = float(max([test.size() for test in results])) - - for test in results: - - # Hyperbolic function. Smaller scores still better - red = 0.999 # minimum factors for each dimension are 1/1000 - test.score = ((test.size() - s_min * red) * - (test.time() - t_min * red)) - - scored.append((test.score, test)) - timed.append((test.time(), test)) - sized.append((test.size(), test)) - #end - - scored.sort() - timed.sort() - sized.sort() - - best_by_size = [] - best_by_time = [] - - pos = 0 - for (score, test) in scored: - pos += 1 - test.score_pos = pos - #end - - scored = [x[1] for x in scored] - - for test in scored: - test.size_pos = PosInAlist(sized, test) - test.time_pos = PosInAlist(timed, test) - #end - - for test in scored: - c = test.config() - s = 0.0 - print 'H-Score: %0.9f %s' % (test.score, test) - #end - - return scored -#end - -def GraphResults(desc, results): - f = open("data-%s.csv" % desc, "w") - for r in results: - f.write("%0.9f\t%d\t# %s\n" % (r.time(), r.size(), r)) - #end - f.close() - os.system("./plot.sh data-%s.csv plot-%s.jpg" % (desc, desc)) -#end - -def GraphSummary(desc, results_ignore): - test_population = 0 - config_ordered = [] - - # drops duplicate test/config pairs (TODO: don't retest them) - for config, cresults in test_all_config_results.items(): - input_config_map = {} - uniq = [] - for test in cresults: - assert test.config() == config - test_population += 1 - key = test.tinput() - if not input_config_map.has_key(key): - input_config_map[key] = {} - #end - if input_config_map[key].has_key(config): - print 'skipping repeat test %s vs. %s' % (input_config_map[key][config], test) - continue - #end - input_config_map[key][config] = test - uniq.append(test) - #end - config_ordered.append(uniq) - #end - - # sort configs descending by number of tests - config_ordered.sort(lambda x, y: len(y) - len(x)) - - print 'population %d: %d configs %d results' % \ - (test_population, - len(config_ordered), - len(config_ordered[0])) - - if config_ordered[0] == 1: - return - #end - - # a map from test-key to test-list w/ various configs - input_set = {} - osize = len(config_ordered) - - for i in xrange(len(config_ordered)): - config = config_ordered[i][0].config() - config_tests = config_ordered[i] - - #print '%s has %d tested inputs' % (config, len(config_tests)) - - if len(input_set) == 0: - input_set = dict([(t.tinput(), [t]) for t in config_tests]) - continue - #end - - # a map from test-key to test-list w/ various configs - update_set = {} - for r in config_tests: - t = r.tinput() - if input_set.has_key(t): - update_set[t] = input_set[t] + [r] - else: - #print 'config %s does not have test %s' % (config, t) - pass - #end - #end - - if len(update_set) <= 1: - break - #end - - input_set = update_set - - # continue if there are more w/ the same number of inputs - if i < (len(config_ordered) - 1) and \ - len(config_ordered[i + 1]) == len(config_tests): - continue - #end - - # synthesize results for multi-test inputs - config_num = None - - # map of config to sum(various test-keys) - smap = {} - for (key, tests) in input_set.items(): - if config_num == None: - # config_num should be the same in all elements - config_num = len(tests) - smap = dict([(r.config(), - (r.time(), - r.size())) - for r in tests]) - else: - # compuate the per-config sum of time/size - assert config_num == len(tests) - smap = dict([(r.config(), - (smap[r.config()][0] + r.time(), - smap[r.config()][1] + r.size())) - for r in tests]) - #end - #end - - if config_num == 1: - continue - #end - - if len(input_set) == osize: - break - #end - - summary = '%s-%d' % (desc, len(input_set)) - osize = len(input_set) - - print 'generate %s w/ %d configs' % (summary, config_num) - syn = [RandomTest(0, (None, None, summary), config, - syntuple = (smap[config][0], smap[config][1])) - for config in smap.keys()] - syn = ScoreTests(syn) - #print 'smap is %s' % (smap,) - #print 'syn is %s' % (' and '.join([str(x) for x in syn])) - GraphResults(summary, syn) - #end -#end - -if __name__ == "__main__": - try: - RunCommand(['rm', '-rf', TMPDIR]) - os.mkdir(TMPDIR) - - rcsf = GetTestRcsFiles() - #generator = rcsf.Generator() - - #sample = SampleDataTest([SAMPLEDIR]) - #generator = sample.Generator() - - #rand = random.Random(135135135135135) - #RunTestLoop(rand, generator, TEST_ROUNDS) - - #RunSpeedTest() - - #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9'])) - x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-S', 'djw'])) - x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-1', '-S', 'djw'])) - #x3r = rcsf.AllPairsByDate(Xdelta3RunClass(['-9', '-T'])) - - #x1r = rcsf.AllPairsByDate(Xdelta1RunClass()) - - except CommandError: - pass - else: - RunCommand(['rm', '-rf', TMPDIR]) - pass - #end -#end diff --git a/xdelta3-second.h b/xdelta3-second.h deleted file mode 100644 index 9096d0f..0000000 --- a/xdelta3-second.h +++ /dev/null @@ -1,315 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2002, 2003, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _XDELTA3_SECOND_H_ -#define _XDELTA3_SECOND_H_ - -static inline void xd3_bit_state_encode_init (bit_state *bits) -{ - bits->cur_byte = 0; - bits->cur_mask = 1; -} - -static inline int xd3_decode_bits (xd3_stream *stream, - bit_state *bits, - const uint8_t **input, - const uint8_t *input_max, - usize_t nbits, - usize_t *valuep) -{ - usize_t value = 0; - usize_t vmask = 1 << nbits; - - if (bits->cur_mask == 0x100) { goto next_byte; } - - for (;;) - { - do - { - vmask >>= 1; - - if (bits->cur_byte & bits->cur_mask) - { - value |= vmask; - } - - bits->cur_mask <<= 1; - - if (vmask == 1) { goto done; } - } - while (bits->cur_mask != 0x100); - - next_byte: - - if (*input == input_max) - { - stream->msg = "secondary decoder end of input"; - return XD3_INTERNAL; - } - - bits->cur_byte = *(*input)++; - bits->cur_mask = 1; - } - - done: - - IF_DEBUG2 (DP(RINT "(d) %u ", value)); - - (*valuep) = value; - return 0; -} - -#if REGRESSION_TEST -/* There may be extra bits at the end of secondary decompression, this macro - * checks for non-zero bits. This is overly strict, but helps pass the - * single-bit-error regression test. */ -static int -xd3_test_clean_bits (xd3_stream *stream, bit_state *bits) -{ - for (; bits->cur_mask != 0x100; bits->cur_mask <<= 1) - { - if (bits->cur_byte & bits->cur_mask) - { - stream->msg = "secondary decoder garbage"; - return XD3_INTERNAL; - } - } - - return 0; -} -#endif - -static xd3_sec_stream* -xd3_get_secondary (xd3_stream *stream, xd3_sec_stream **sec_streamp) -{ - xd3_sec_stream *sec_stream; - - if ((sec_stream = *sec_streamp) == NULL) - { - if ((*sec_streamp = stream->sec_type->alloc (stream)) == NULL) - { - return NULL; - } - - sec_stream = *sec_streamp; - - /* If cuumulative stats, init once. */ - stream->sec_type->init (sec_stream); - } - - return sec_stream; -} - -static int -xd3_decode_secondary (xd3_stream *stream, - xd3_desect *sect, - xd3_sec_stream **sec_streamp) -{ - xd3_sec_stream *sec_stream; - uint32_t dec_size; - uint8_t *out_used; - int ret; - - if ((sec_stream = xd3_get_secondary (stream, sec_streamp)) == NULL) - { - return ENOMEM; - } - - /* Decode the size, allocate the buffer. */ - if ((ret = xd3_read_size (stream, & sect->buf, - sect->buf_max, & dec_size)) || - (ret = xd3_decode_allocate (stream, dec_size, - & sect->copied2, & sect->alloc2))) - { - return ret; - } - - out_used = sect->copied2; - - if ((ret = stream->sec_type->decode (stream, sec_stream, - & sect->buf, sect->buf_max, - & out_used, out_used + dec_size))) - { - return ret; - } - - if (sect->buf != sect->buf_max) - { - stream->msg = "secondary decoder finished with unused input"; - return XD3_INTERNAL; - } - - if (out_used != sect->copied2 + dec_size) - { - stream->msg = "secondary decoder short output"; - return XD3_INTERNAL; - } - - sect->buf = sect->copied2; - sect->buf_max = sect->copied2 + dec_size; - sect->size = dec_size; - - return 0; -} - -#if XD3_ENCODER -static inline int xd3_encode_bit (xd3_stream *stream, - xd3_output **output, - bit_state *bits, - int bit) -{ - int ret; - - if (bit) - { - bits->cur_byte |= bits->cur_mask; - } - - /* OPT: Might help to buffer more than 8 bits at once. */ - if (bits->cur_mask == 0x80) - { - if ((ret = xd3_emit_byte (stream, output, bits->cur_byte)) != 0) - { - return ret; - } - - bits->cur_mask = 1; - bits->cur_byte = 0; - } - else - { - bits->cur_mask <<= 1; - } - - return 0; -} - -static inline int xd3_flush_bits (xd3_stream *stream, - xd3_output **output, - bit_state *bits) -{ - return (bits->cur_mask == 1) ? 0 : - xd3_emit_byte (stream, output, bits->cur_byte); -} - -static inline int xd3_encode_bits (xd3_stream *stream, - xd3_output **output, - bit_state *bits, - usize_t nbits, - usize_t value) -{ - int ret; - usize_t mask = 1 << nbits; - - XD3_ASSERT (nbits > 0); - XD3_ASSERT (nbits < sizeof (usize_t) * 8); - XD3_ASSERT (value < mask); - - do - { - mask >>= 1; - - if ((ret = xd3_encode_bit (stream, output, bits, value & mask))) - { - return ret; - } - } - while (mask != 1); - - IF_DEBUG2 (DP(RINT "(e) %u ", value)); - - return 0; -} - -static int -xd3_encode_secondary (xd3_stream *stream, - xd3_output **head, - xd3_output **tail, - xd3_sec_stream **sec_streamp, - xd3_sec_cfg *cfg, - int *did_it) -{ - xd3_sec_stream *sec_stream; - xd3_output *tmp_head; - xd3_output *tmp_tail; - - usize_t comp_size; - usize_t orig_size; - - int ret; - - orig_size = xd3_sizeof_output (*head); - - if (orig_size < SECONDARY_MIN_INPUT) { return 0; } - - if ((sec_stream = xd3_get_secondary (stream, sec_streamp)) == NULL) - { - return ENOMEM; - } - - tmp_head = xd3_alloc_output (stream, NULL); - - /* Encode the size, encode the data. Encoding the size makes it - * simpler, but is a little gross. Should not need the entire - * section in contiguous memory, but it is much easier this way. */ - if ((ret = xd3_emit_size (stream, & tmp_head, orig_size)) || - (ret = stream->sec_type->encode (stream, sec_stream, *head, - tmp_head, cfg))) - { - goto getout; - } - - /* If the secondary compressor determines it's no good, it returns - * XD3_NOSECOND. */ - - /* Setup tmp_tail, comp_size */ - tmp_tail = tmp_head; - comp_size = tmp_head->next; - - while (tmp_tail->next_page != NULL) - { - tmp_tail = tmp_tail->next_page; - comp_size += tmp_tail->next; - } - - XD3_ASSERT (comp_size == xd3_sizeof_output (tmp_head)); - XD3_ASSERT (tmp_tail != NULL); - - if (comp_size < (orig_size - SECONDARY_MIN_SAVINGS)) - { - IF_DEBUG1(DP(RINT "secondary saved %u bytes: %u -> %u (%0.2f%%)\n", - orig_size - comp_size, orig_size, comp_size, - 100.0 * (double) comp_size / (double) orig_size)); - - xd3_free_output (stream, *head); - - *head = tmp_head; - *tail = tmp_tail; - *did_it = 1; - } - else - { - getout: - if (ret == XD3_NOSECOND) { ret = 0; } - xd3_free_output (stream, tmp_head); - } - - return ret; -} -#endif /* XD3_ENCODER */ -#endif /* _XDELTA3_SECOND_H_ */ diff --git a/xdelta3-test.h b/xdelta3-test.h deleted file mode 100644 index 979683f..0000000 --- a/xdelta3-test.h +++ /dev/null @@ -1,2827 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* This is public-domain Mersenne Twister code, - * attributed to Michael Brundage. Thanks! - * http://www.qbrundage.com/michaelb/pubs/essays/random_number_generation.html - */ -static const uint32_t TEST_SEED1 = 5489UL; -#define MT_LEN 624 -#define MT_IA 397 -static const uint32_t UPPER_MASK = 0x80000000; -static const uint32_t LOWER_MASK = 0x7FFFFFFF; -static const uint32_t MATRIX_A = 0x9908B0DF; - -typedef struct mtrand mtrand; - -struct mtrand { - int mt_index_; - uint32_t mt_buffer_[MT_LEN]; -}; - -void mt_init(mtrand *mt, uint32_t seed) { - int i; - mt->mt_buffer_[0] = seed; - mt->mt_index_ = MT_LEN; - for (i = 1; i < MT_LEN; i++) { - /* See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. */ - /* In the previous versions, MSBs of the seed affect */ - /* only MSBs of the array mt[]. */ - /* 2002/01/09 modified by Makoto Matsumoto */ - mt->mt_buffer_[i] = - (1812433253UL * (mt->mt_buffer_[i-1] ^ (mt->mt_buffer_[i-1] >> 30)) + i); - } -} - - -uint32_t mt_random (mtrand *mt) { - uint32_t y; - unsigned long mag01[2]; - mag01[0] = 0; - mag01[1] = MATRIX_A; - - if (mt->mt_index_ >= MT_LEN) { - int kk; - - for (kk = 0; kk < MT_LEN - MT_IA; kk++) { - y = (mt->mt_buffer_[kk] & UPPER_MASK) | (mt->mt_buffer_[kk + 1] & LOWER_MASK); - mt->mt_buffer_[kk] = mt->mt_buffer_[kk + MT_IA] ^ (y >> 1) ^ mag01[y & 0x1UL]; - } - for (;kk < MT_LEN - 1; kk++) { - y = (mt->mt_buffer_[kk] & UPPER_MASK) | (mt->mt_buffer_[kk + 1] & LOWER_MASK); - mt->mt_buffer_[kk] = mt->mt_buffer_[kk + (MT_IA - MT_LEN)] ^ (y >> 1) ^ mag01[y & 0x1UL]; - } - y = (mt->mt_buffer_[MT_LEN - 1] & UPPER_MASK) | (mt->mt_buffer_[0] & LOWER_MASK); - mt->mt_buffer_[MT_LEN - 1] = mt->mt_buffer_[MT_IA - 1] ^ (y >> 1) ^ mag01[y & 0x1UL]; - mt->mt_index_ = 0; - } - - y = mt->mt_buffer_[mt->mt_index_++]; - - y ^= (y >> 11); - y ^= (y << 7) & 0x9d2c5680UL; - y ^= (y << 15) & 0xefc60000UL; - y ^= (y >> 18); - - return y; -} - -static mtrand static_mtrand; - -#include <math.h> - -static uint32_t -mt_exp_rand (uint32_t mean, uint32_t max_value) -{ - double mean_d = mean; - double erand = log (1.0 / (mt_random (&static_mtrand) / - (double)UINT32_MAX)); - uint32_t x = (uint32_t) (mean_d * erand + 0.5); - - return min (x, max_value); -} - -#ifndef WIN32 -#include <sys/wait.h> -#endif - -#define MSG_IS(x) (stream->msg != NULL && strcmp ((x), stream->msg) == 0) - -static const usize_t TWO_MEGS_AND_DELTA = (2 << 20) + (1 << 10); -static const usize_t ADDR_CACHE_ROUNDS = 10000; - -static const usize_t TEST_FILE_MEAN = 16384; -static const double TEST_ADD_MEAN = 128; -static const double TEST_ADD_MAX = 512; -static const double TEST_ADD_RATIO = 0.1; -static const double TEST_EPSILON = 0.25; - -#define TESTBUFSIZE (1024 * 16) - -#define TESTFILESIZE (1024) - -static char TEST_TARGET_FILE[TESTFILESIZE]; -static char TEST_SOURCE_FILE[TESTFILESIZE]; -static char TEST_DELTA_FILE[TESTFILESIZE]; -static char TEST_RECON_FILE[TESTFILESIZE]; -static char TEST_RECON2_FILE[TESTFILESIZE]; -static char TEST_COPY_FILE[TESTFILESIZE]; -static char TEST_NOPERM_FILE[TESTFILESIZE]; - -#define CHECK(cond) if (!(cond)) { DP(RINT "check failure: " #cond); abort(); } - -/* Use a fixed soft config so that test values are fixed. See also - * test_compress_text(). */ -static const char* test_softcfg_str = "-C9,3,4,8,2,36,70"; - -static int test_setup (void); - -/*********************************************************************** - TEST HELPERS - ***********************************************************************/ - -static void DOT (void) { DP(RINT "."); } -static int do_cmd (xd3_stream *stream, const char *buf) -{ - int ret; - if ((ret = system (buf)) != 0) - { - if (WIFEXITED (ret)) - { - stream->msg = "command exited non-zero"; - } - else - { - stream->msg = "abnormal command termination"; - } - return XD3_INTERNAL; - } - return 0; -} - -static int do_fail (xd3_stream *stream, const char *buf) -{ - int ret; - ret = system (buf); - if (! WIFEXITED (ret) || WEXITSTATUS (ret) != 1) - { - stream->msg = "command should have not succeeded"; - DP(RINT "command was %s", buf); - return XD3_INTERNAL; - } - return 0; -} - -/* Test that the exponential distribution actually produces its mean. */ -static int -test_random_numbers (xd3_stream *stream, int ignore) -{ - usize_t i; - usize_t sum = 0; - usize_t mean = 50; - usize_t n_rounds = 1000000; - double average, error; - double allowed_error = 0.1; - - mt_init (& static_mtrand, 0x9f73f7fe); - - for (i = 0; i < n_rounds; i += 1) - { - sum += mt_exp_rand (mean, USIZE_T_MAX); - } - - average = (double) sum / (double) n_rounds; - error = average - (double) mean; - - if (error < allowed_error && error > -allowed_error) - { - return 0; - } - - /*DP(RINT "error is %f\n", error);*/ - stream->msg = "random distribution looks broken"; - return XD3_INTERNAL; -} - -static void -test_unlink (char* file) -{ - char buf[TESTBUFSIZE]; - while (unlink (file) != 0) - { - if (errno == ENOENT) - { - break; - } - sprintf (buf, "rm -f %s", file); - system (buf); - } -} - -static void -test_cleanup (void) -{ - test_unlink (TEST_TARGET_FILE); - test_unlink (TEST_SOURCE_FILE); - test_unlink (TEST_DELTA_FILE); - test_unlink (TEST_RECON_FILE); - test_unlink (TEST_RECON2_FILE); - test_unlink (TEST_COPY_FILE); - test_unlink (TEST_NOPERM_FILE); -} - -static int -test_setup (void) -{ - static int x = 0; - x++; - sprintf (TEST_TARGET_FILE, "/tmp/xdtest.target.%d", x); - sprintf (TEST_SOURCE_FILE, "/tmp/xdtest.source.%d", x); - sprintf (TEST_DELTA_FILE, "/tmp/xdtest.delta.%d", x); - sprintf (TEST_RECON_FILE, "/tmp/xdtest.recon.%d", x); - sprintf (TEST_RECON2_FILE, "/tmp/xdtest.recon2.%d", x); - sprintf (TEST_COPY_FILE, "/tmp/xdtest.copy.%d", x); - sprintf (TEST_NOPERM_FILE, "/tmp/xdtest.noperm.%d", x); - test_cleanup(); - return 0; -} - -static int -test_make_inputs (xd3_stream *stream, xoff_t *ss_out, xoff_t *ts_out) -{ - usize_t ts = (mt_random (&static_mtrand) % TEST_FILE_MEAN) + TEST_FILE_MEAN / 2; - usize_t ss = (mt_random (&static_mtrand) % TEST_FILE_MEAN) + TEST_FILE_MEAN / 2; - uint8_t *buf = (uint8_t*) malloc (ts + ss), *sbuf = buf, *tbuf = buf + ss; - usize_t sadd = 0, sadd_max = ss * TEST_ADD_RATIO; - FILE *tf = NULL, *sf = NULL; - usize_t i, j; - int ret; - - if (buf == NULL) { return ENOMEM; } - - if ((tf = fopen (TEST_TARGET_FILE, "w")) == NULL || - (ss_out != NULL && (sf = fopen (TEST_SOURCE_FILE, "w")) == NULL)) - { - stream->msg = "write failed"; - ret = get_errno (); - goto failure; - } - - if (ss_out != NULL) - { - for (i = 0; i < ss; ) - { - sbuf[i++] = mt_random (&static_mtrand); - } - } - - /* Then modify the data to produce copies, everything not copied is - * an add. The following logic produces the TEST_ADD_RATIO. The - * variable SADD contains the number of adds so far, which should - * not exceed SADD_MAX. */ - - /* DP(RINT "ss = %u ts = %u\n", ss, ts); */ - for (i = 0; i < ts; ) - { - size_t left = ts - i; - size_t next = mt_exp_rand (TEST_ADD_MEAN, TEST_ADD_MAX); - size_t add_left = sadd_max - sadd; - double add_prob = (left == 0) ? 0 : (add_left / (double) left); - int do_copy; - - next = min (left, next); - do_copy = (next > add_left || (mt_random (&static_mtrand) / (double)USIZE_T_MAX) >= add_prob); - - if (ss_out == NULL) - { - do_copy &= (i > 0); - } - else - { - do_copy &= (ss - next) > 0; - } - - if (do_copy) - { - /* Copy */ - size_t offset = mt_random (&static_mtrand) % ((ss_out == NULL) ? i : (ss - next)); - /* DP(RINT "[%u] copy %u at %u ", i, next, offset); */ - - for (j = 0; j < next; j += 1) - { - char c = ((ss_out == NULL) ? tbuf : sbuf)[offset + j]; - /* DP(RINT "%x%x", (c >> 4) & 0xf, c & 0xf); */ - tbuf[i++] = c; - } - /* DP(RINT "\n"); */ - } - else - { - /* Add */ - /* DP(RINT "[%u] add %u ", i, next); */ - for (j = 0; j < next; j += 1) - { - char c = mt_random (&static_mtrand); - /* DP(RINT "%x%x", (c >> 4) & 0xf, c & 0xf); */ - tbuf[i++] = c; - } - /* DP(RINT "\n"); */ - sadd += next; - } - } - - /* DP(RINT "sadd = %u max = %u\n", sadd, sadd_max); */ - - if ((fwrite (tbuf, 1, ts, tf) != ts) || - (ss_out != NULL && (fwrite (sbuf, 1, ss, sf) != ss))) - { - stream->msg = "write failed"; - ret = get_errno (); - goto failure; - } - - if ((ret = fclose (tf)) || (ss_out != NULL && (ret = fclose (sf)))) - { - stream->msg = "close failed"; - ret = get_errno (); - goto failure; - } - - if (ts_out) { (*ts_out) = ts; } - if (ss_out) { (*ss_out) = ss; } - - failure: - free (buf); - return ret; -} - -static int -compare_files (xd3_stream *stream, const char* tgt, const char *rec) -{ - FILE *orig, *recons; - static uint8_t obuf[TESTBUFSIZE], rbuf[TESTBUFSIZE]; - int offset = 0; - int i; - int oc, rc; - - if ((orig = fopen (tgt, "r")) == NULL) - { - DP(RINT "open %s failed", tgt); - stream->msg = "open failed"; - return get_errno (); - } - - if ((recons = fopen (rec, "r")) == NULL) - { - DP(RINT "open %s failed", rec); - stream->msg = "open failed"; - return get_errno (); - } - - for (;;) - { - oc = fread (obuf, 1, TESTBUFSIZE, orig); - rc = fread (rbuf, 1, TESTBUFSIZE, recons); - - if (oc < 0 || rc < 0) - { - stream->msg = "read failed"; - return get_errno (); - } - - if (oc != rc) - { - stream->msg = "compare files: different length"; - return XD3_INTERNAL; - } - - if (oc == 0) - { - break; - } - - for (i = 0; i < oc; i += 1) - { - if (obuf[i] != rbuf[i]) - { - stream->msg = "compare files: different values"; - return XD3_INTERNAL; - } - } - - offset += oc; - } - - fclose (orig); - fclose (recons); - return 0; -} - -static int -test_save_copy (const char *origname) -{ - char buf[TESTBUFSIZE]; - int ret; - - sprintf (buf, "cp -f %s %s", origname, TEST_COPY_FILE); - - if ((ret = system (buf)) != 0) - { - return XD3_INTERNAL; - } - - return 0; -} - -static int -test_file_size (const char* file, xoff_t *size) -{ - struct stat sbuf; - int ret; - (*size) = 0; - - if (stat (file, & sbuf) < 0) - { - ret = get_errno (); - DP(RINT "xdelta3: stat failed: %s: %s\n", file, strerror (ret)); - return ret; - } - - if (! S_ISREG (sbuf.st_mode)) - { - ret = XD3_INTERNAL; - DP(RINT "xdelta3: not a regular file: %s: %s\n", file, strerror (ret)); - return ret; - } - - (*size) = sbuf.st_size; - return 0; -} - -/*********************************************************************** - READ OFFSET - ***********************************************************************/ - -/* Common test for read_integer errors: encodes a 64-bit value and - * then attempts to read as a 32-bit value. If TRUNC is non-zero, - * attempts to get errors by shortening the input, otherwise it should - * overflow. Expects XD3_INTERNAL and MSG. */ -static int -test_read_integer_error (xd3_stream *stream, usize_t trunto, const char *msg) -{ - uint64_t eval = 1ULL << 34; - uint32_t rval; - xd3_output *buf = NULL; - const uint8_t *max; - const uint8_t *inp; - int ret; - - buf = xd3_alloc_output (stream, buf); - - if ((ret = xd3_emit_uint64_t (stream, & buf, eval))) - { - goto fail; - } - - again: - - inp = buf->base; - max = buf->base + buf->next - trunto; - - if ((ret = xd3_read_uint32_t (stream, & inp, max, & rval)) != XD3_INVALID_INPUT || - !MSG_IS (msg)) - { - ret = XD3_INTERNAL; - } - else if (trunto && trunto < buf->next) - { - trunto += 1; - goto again; - } - else - { - ret = 0; - } - - fail: - xd3_free_output (stream, buf); - return ret; -} - -/* Test integer overflow using the above routine. */ -static int -test_decode_integer_overflow (xd3_stream *stream, int unused) -{ - return test_read_integer_error (stream, 0, "overflow in read_intger"); -} - -/* Test integer EOI using the above routine. */ -static int -test_decode_integer_end_of_input (xd3_stream *stream, int unused) -{ - return test_read_integer_error (stream, 1, "end-of-input in read_integer"); -} - -/* Test that emit_integer/decode_integer/sizeof_integer/read_integer - * work on correct inputs. Tests powers of (2^7), plus or minus, up - * to the maximum value. */ -#define TEST_ENCODE_DECODE_INTEGER(TYPE,ONE,MAX) \ - xd3_output *rbuf = NULL; \ - xd3_output *dbuf = NULL; \ - TYPE values[64]; \ - usize_t nvalues = 0; \ - usize_t i; \ - int ret = 0; \ - \ - for (i = 0; i < (sizeof (TYPE) * 8); i += 7) \ - { \ - values[nvalues++] = (ONE << i) - ONE; \ - values[nvalues++] = (ONE << i); \ - values[nvalues++] = (ONE << i) + ONE; \ - } \ - \ - values[nvalues++] = MAX-ONE; \ - values[nvalues++] = MAX; \ - \ - rbuf = xd3_alloc_output (stream, rbuf); \ - dbuf = xd3_alloc_output (stream, dbuf); \ - \ - for (i = 0; i < nvalues; i += 1) \ - { \ - const uint8_t *max; \ - const uint8_t *inp; \ - TYPE val; \ - \ - DOT (); \ - rbuf->next = 0; \ - \ - if ((ret = xd3_emit_ ## TYPE (stream, & rbuf, values[i])) || \ - (ret = xd3_emit_ ## TYPE (stream, & dbuf, values[i]))) \ - { \ - goto fail; \ - } \ - \ - inp = rbuf->base; \ - max = rbuf->base + rbuf->next; \ - \ - if (rbuf->next != xd3_sizeof_ ## TYPE (values[i])) \ - { \ - ret = XD3_INTERNAL; \ - goto fail; \ - } \ - \ - if ((ret = xd3_read_ ## TYPE (stream, & inp, max, & val))) \ - { \ - goto fail; \ - } \ - \ - if (val != values[i]) \ - { \ - ret = XD3_INTERNAL; \ - goto fail; \ - } \ - \ - DOT (); \ - } \ - \ - stream->next_in = dbuf->base; \ - stream->avail_in = dbuf->next; \ - \ - for (i = 0; i < nvalues; i += 1) \ - { \ - TYPE val; \ - \ - if ((ret = xd3_decode_ ## TYPE (stream, & val))) \ - { \ - goto fail; \ - } \ - \ - if (val != values[i]) \ - { \ - ret = XD3_INTERNAL; \ - goto fail; \ - } \ - } \ - \ - if (stream->avail_in != 0) \ - { \ - ret = XD3_INTERNAL; \ - goto fail; \ - } \ - \ - fail: \ - xd3_free_output (stream, rbuf); \ - xd3_free_output (stream, dbuf); \ - \ - return ret - -static int -test_encode_decode_uint32_t (xd3_stream *stream, int unused) -{ - TEST_ENCODE_DECODE_INTEGER(uint32_t,1U,UINT32_MAX); -} - -static int -test_encode_decode_uint64_t (xd3_stream *stream, int unused) -{ - TEST_ENCODE_DECODE_INTEGER(uint64_t,1ULL,UINT64_MAX); -} - -static int -test_usize_t_overflow (xd3_stream *stream, int unused) -{ - if (USIZE_T_OVERFLOW (0, 0)) { goto fail; } - if (USIZE_T_OVERFLOW (USIZE_T_MAX, 0)) { goto fail; } - if (USIZE_T_OVERFLOW (0, USIZE_T_MAX)) { goto fail; } - if (USIZE_T_OVERFLOW (USIZE_T_MAX / 2, 0)) { goto fail; } - if (USIZE_T_OVERFLOW (USIZE_T_MAX / 2, USIZE_T_MAX / 2)) { goto fail; } - if (USIZE_T_OVERFLOW (USIZE_T_MAX / 2, USIZE_T_MAX / 2 + 1)) { goto fail; } - - if (! USIZE_T_OVERFLOW (USIZE_T_MAX, 1)) { goto fail; } - if (! USIZE_T_OVERFLOW (1, USIZE_T_MAX)) { goto fail; } - if (! USIZE_T_OVERFLOW (USIZE_T_MAX / 2 + 1, USIZE_T_MAX / 2 + 1)) { goto fail; } - - return 0; - - fail: - stream->msg = "incorrect overflow computation"; - return XD3_INTERNAL; -} - -static int -test_forward_match (xd3_stream *stream, int unused) -{ - usize_t i; - uint8_t buf1[256], buf2[256]; - - memset(buf1, 0, 256); - memset(buf2, 0, 256); - - for (i = 0; i < 256; i++) - { - CHECK(xd3_forward_match(buf1, buf2, i) == i); - } - - for (i = 0; i < 255; i++) - { - buf2[i] = 1; - CHECK(xd3_forward_match(buf1, buf2, 256) == i); - buf2[i] = 0; - } - - return 0; -} - -/*********************************************************************** - Address cache - ***********************************************************************/ - -static int -test_address_cache (xd3_stream *stream, int unused) -{ - int ret; - usize_t i; - usize_t offset; - usize_t *addrs; - uint8_t *big_buf, *buf_max; - const uint8_t *buf; - xd3_output *outp; - uint8_t *modes; - int mode_counts[16]; - - stream->acache.s_near = stream->code_table_desc->near_modes; - stream->acache.s_same = stream->code_table_desc->same_modes; - - if ((ret = xd3_encode_init_partial (stream))) { return ret; } - - addrs = (usize_t*) xd3_alloc (stream, sizeof (usize_t), ADDR_CACHE_ROUNDS); - modes = (uint8_t*) xd3_alloc (stream, sizeof (uint8_t), ADDR_CACHE_ROUNDS); - - memset (mode_counts, 0, sizeof (mode_counts)); - memset (modes, 0, ADDR_CACHE_ROUNDS); - - addrs[0] = 0; - - mt_init (& static_mtrand, 0x9f73f7fc); - - /* First pass: encode addresses */ - xd3_init_cache (& stream->acache); - - for (offset = 1; offset < ADDR_CACHE_ROUNDS; offset += 1) - { - double p; - usize_t addr; - usize_t prev_i; - usize_t nearby; - - p = (mt_random (&static_mtrand) / (double)USIZE_T_MAX); - prev_i = mt_random (&static_mtrand) % offset; - nearby = (mt_random (&static_mtrand) % 256) % offset; - nearby = max (1U, nearby); - - if (p < 0.1) { addr = addrs[offset-nearby]; } - else if (p < 0.4) { addr = min (addrs[prev_i] + nearby, offset-1); } - else { addr = prev_i; } - - if ((ret = xd3_encode_address (stream, addr, offset, & modes[offset]))) { return ret; } - - addrs[offset] = addr; - mode_counts[modes[offset]] += 1; - } - - /* Copy addresses into a contiguous buffer. */ - big_buf = (uint8_t*) xd3_alloc (stream, xd3_sizeof_output (ADDR_HEAD (stream)), 1); - - for (offset = 0, outp = ADDR_HEAD (stream); outp != NULL; offset += outp->next, outp = outp->next_page) - { - memcpy (big_buf + offset, outp->base, outp->next); - } - - buf_max = big_buf + offset; - buf = big_buf; - - /* Second pass: decode addresses */ - xd3_init_cache (& stream->acache); - - for (offset = 1; offset < ADDR_CACHE_ROUNDS; offset += 1) - { - uint32_t addr; - - if ((ret = xd3_decode_address (stream, offset, modes[offset], & buf, buf_max, & addr))) { return ret; } - - if (addr != addrs[offset]) - { - stream->msg = "incorrect decoded address"; - return XD3_INTERNAL; - } - } - - /* Check that every byte, mode was used. */ - if (buf != buf_max) - { - stream->msg = "address bytes not used"; - return XD3_INTERNAL; - } - - for (i = 0; i < (2 + stream->acache.s_same + stream->acache.s_near); i += 1) - { - if (mode_counts[i] == 0) - { - stream->msg = "address mode not used"; - return XD3_INTERNAL; - } - } - - xd3_free (stream, modes); - xd3_free (stream, addrs); - xd3_free (stream, big_buf); - - return 0; -} - -/*********************************************************************** - Encode and decode with single bit error - ***********************************************************************/ - -/* It compresses from 256 to around 185 bytes. - * Avoids matching addresses that are a single-bit difference. - * Avoids matching address 0. */ -static const uint8_t test_text[] = -"this is a story\n" -"abouttttttttttt\n" -"- his is a stor\n" -"- about nothing " -" all. boutique -" -"his story is a -" -"about " -"what happens all" -" the time what -" -"am I ttttttt the" -" person said, so" -" what, per son -" -" gory story is -" -" about nothing -" -"tttttt to test -" -"his sto nothing"; - -static const uint8_t test_apphead[] = "header test"; - -static int -test_compress_text (xd3_stream *stream, - uint8_t *encoded, - usize_t *encoded_size) -{ - int ret; - xd3_config cfg; - int oflags = stream->flags; - int flags = stream->flags | XD3_FLUSH; - - xd3_free_stream (stream); - xd3_init_config (& cfg, flags); - - /* This configuration is fixed so that the "expected non-error" the counts in - * decompress_single_bit_errors are too. See test_coftcfg_str. */ - cfg.smatch_cfg = XD3_SMATCH_SOFT; - cfg.smatcher_soft.name = "test"; - cfg.smatcher_soft.large_look = 64; /* no source, not used */ - cfg.smatcher_soft.large_step = 64; /* no source, not used */ - cfg.smatcher_soft.small_look = 4; - cfg.smatcher_soft.small_chain = 128; - cfg.smatcher_soft.small_lchain = 16; - cfg.smatcher_soft.max_lazy = 8; - cfg.smatcher_soft.long_enough = 128; - - xd3_config_stream (stream, & cfg); - - (*encoded_size) = 0; - - xd3_set_appheader (stream, test_apphead, strlen ((char*) test_apphead)); - - if ((ret = xd3_encode_stream (stream, test_text, sizeof (test_text), - encoded, encoded_size, 4*sizeof (test_text)))) { goto fail; } - - if ((ret = xd3_close_stream (stream))) { goto fail; } - - fail: - xd3_free_stream (stream); - xd3_init_config (& cfg, oflags); - xd3_config_stream (stream, & cfg); - return ret; -} - -static int -test_decompress_text (xd3_stream *stream, uint8_t *enc, usize_t enc_size, usize_t test_desize) -{ - xd3_config cfg; - char decoded[sizeof (test_text)]; - uint8_t *apphead; - usize_t apphead_size; - usize_t decoded_size; - const char *msg; - int ret; - usize_t pos = 0; - int flags = stream->flags; - usize_t take; - - input: - /* Test decoding test_desize input bytes at a time */ - take = min (enc_size - pos, test_desize); - CHECK(take > 0); - - xd3_avail_input (stream, enc + pos, take); - again: - ret = xd3_decode_input (stream); - - pos += take; - take = 0; - - switch (ret) - { - case XD3_OUTPUT: - break; - case XD3_WINSTART: - case XD3_GOTHEADER: - goto again; - case XD3_INPUT: - if (pos < enc_size) { goto input; } - /* else fallthrough */ - case XD3_WINFINISH: - default: - goto fail; - } - - CHECK(ret == XD3_OUTPUT); - CHECK(pos == enc_size); - - if (stream->avail_out != sizeof (test_text)) - { - stream->msg = "incorrect output size"; - ret = XD3_INTERNAL; - goto fail; - } - - decoded_size = stream->avail_out; - memcpy (decoded, stream->next_out, stream->avail_out); - - xd3_consume_output (stream); - - if ((ret = xd3_get_appheader (stream, & apphead, & apphead_size))) { goto fail; } - - if (apphead_size != strlen ((char*) test_apphead) || - memcmp (apphead, test_apphead, strlen ((char*) test_apphead)) != 0) - { - stream->msg = "incorrect appheader"; - ret = XD3_INTERNAL; - goto fail; - } - - if ((ret = xd3_decode_input (stream)) != XD3_WINFINISH || - (ret = xd3_close_stream (stream)) != 0) - { - goto fail; - } - - if (decoded_size != sizeof (test_text) || - memcmp (decoded, test_text, sizeof (test_text)) != 0) - { - stream->msg = "incorrect output text"; - ret = EIO; - } - - fail: - msg = stream->msg; - xd3_free_stream (stream); - xd3_init_config (& cfg, flags); - xd3_config_stream (stream, & cfg); - stream->msg = msg; - - return ret; -} - -static int -test_decompress_single_bit_error (xd3_stream *stream, int expected_non_failures) -{ - int ret; - usize_t i; - uint8_t encoded[4*sizeof (test_text)]; /* make room for alt code table */ - usize_t encoded_size; - int non_failures = 0; - int cksum = (stream->flags & XD3_ADLER32) != 0; - -//#define DEBUG_TEST_FAILURES -#ifndef DEBUG_TEST_FAILURES -#define TEST_FAILURES() -#else - /* For checking non-failure cases by hand, enable this macro and run - * xdelta printdelta with print_cpymode disabled. Every non-failure - * should change a copy address mode, which doesn't cause a failure - * because the address cache starts out with all zeros. - - ./xdelta3 test - for i in test_text.xz.*; do ./xdelta3 printdelta $i > $i.out; - diff $i.out test_text.xz.0.out; done - - */ - system ("rm -rf test_text.*"); - { - char buf[TESTBUFSIZE]; - FILE *f; - sprintf (buf, "test_text"); - f = fopen (buf, "w"); - fwrite (test_text,1,sizeof (test_text),f); - fclose (f); - } -#define TEST_FAILURES() \ - do { \ - char buf[TESTBUFSIZE]; \ - FILE *f; \ - sprintf (buf, "test_text.xz.%d", non_failures); \ - f = fopen (buf, "w"); \ - fwrite (encoded,1,encoded_size,f); \ - fclose (f); \ - } while (0) -#endif - - stream->sec_data.inefficient = 1; - stream->sec_inst.inefficient = 1; - stream->sec_addr.inefficient = 1; - - /* Encode text, test correct input */ - if ((ret = test_compress_text (stream, encoded, & encoded_size))) - { - /*stream->msg = "without error: encode failure";*/ - return ret; - } - - if ((ret = test_decompress_text (stream, encoded, encoded_size, - sizeof (test_text) / 4))) - { - /*stream->msg = "without error: decode failure";*/ - return ret; - } - - TEST_FAILURES(); - - for (i = 0; i < encoded_size*8; i += 1) - { - /* Single bit error. */ - encoded[i/8] ^= 1 << (i%8); - - if ((ret = test_decompress_text (stream, encoded, - encoded_size, sizeof (test_text))) == 0) - { - non_failures += 1; -#ifdef DEBUG_TEST_FAILURES - DP(RINT "%u[%u] non-failure %u\n", i/8, i%8, non_failures); -#endif - TEST_FAILURES(); - } - else - { - /*DP(RINT "%u[%u] failure: %s\n", i/8, i%8, stream->msg);*/ - } - - /* decompress_text returns EIO when the final memcmp() fails, but that - * should never happen with checksumming on. */ - if (cksum && ret == EIO) - { - /*DP(RINT "%u[%u] cksum mismatch\n", i/8, i%8);*/ - stream->msg = "checksum mismatch"; - return XD3_INTERNAL; - } - - /* Undo single bit error. */ - encoded[i/8] ^= 1 << (i%8); - } - - /* Test correct input again */ - if ((ret = test_decompress_text (stream, encoded, encoded_size, 1))) - { - /*stream->msg = "without error: decode failure";*/ - return ret; - } - - /* Check expected non-failures */ - if (non_failures != expected_non_failures) - { - DP(RINT "non-failures %u; expected %u", - non_failures, expected_non_failures); - stream->msg = "incorrect"; - return XD3_INTERNAL; - } - - DOT (); - - return 0; -} - -/*********************************************************************** - Secondary compression tests - ***********************************************************************/ - -#if SECONDARY_ANY -typedef int (*sec_dist_func) (xd3_stream *stream, xd3_output *data); - -static int sec_dist_func1 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func2 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func3 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func4 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func5 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func6 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func7 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func8 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func9 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func10 (xd3_stream *stream, xd3_output *data); -static int sec_dist_func11 (xd3_stream *stream, xd3_output *data); - -static sec_dist_func sec_dists[] = -{ - sec_dist_func1, - sec_dist_func2, - sec_dist_func3, - sec_dist_func4, - sec_dist_func5, - sec_dist_func6, - sec_dist_func7, - sec_dist_func8, - sec_dist_func9, - sec_dist_func10, - sec_dist_func11, -}; - -/* Test ditsribution: 100 bytes of the same character (13). */ -static int -sec_dist_func1 (xd3_stream *stream, xd3_output *data) -{ - int i, ret; - for (i = 0; i < 100; i += 1) - { - if ((ret = xd3_emit_byte (stream, & data, 13))) { return ret; } - } - return 0; -} - -/* Test ditsribution: uniform covering half the alphabet. */ -static int -sec_dist_func2 (xd3_stream *stream, xd3_output *data) -{ - int i, ret; - for (i = 0; i < ALPHABET_SIZE; i += 1) - { - if ((ret = xd3_emit_byte (stream, & data, i%(ALPHABET_SIZE/2)))) { return ret; } - } - return 0; -} - -/* Test ditsribution: uniform covering the entire alphabet. */ -static int -sec_dist_func3 (xd3_stream *stream, xd3_output *data) -{ - int i, ret; - for (i = 0; i < ALPHABET_SIZE; i += 1) - { - if ((ret = xd3_emit_byte (stream, & data, i%ALPHABET_SIZE))) { return ret; } - } - return 0; -} - -/* Test distribution: An exponential distribution covering half the alphabet */ -static int -sec_dist_func4 (xd3_stream *stream, xd3_output *data) -{ - int i, ret, x; - for (i = 0; i < ALPHABET_SIZE*20; i += 1) - { - x = mt_exp_rand (10, ALPHABET_SIZE/2); - if ((ret = xd3_emit_byte (stream, & data, x))) { return ret; } - } - return 0; -} - -/* Test distribution: An exponential distribution covering the entire alphabet */ -static int -sec_dist_func5 (xd3_stream *stream, xd3_output *data) -{ - int i, ret, x; - for (i = 0; i < ALPHABET_SIZE*20; i += 1) - { - x = mt_exp_rand (10, ALPHABET_SIZE-1); - if ((ret = xd3_emit_byte (stream, & data, x))) { return ret; } - } - return 0; -} - -/* Test distribution: An uniform random distribution covering half the alphabet */ -static int -sec_dist_func6 (xd3_stream *stream, xd3_output *data) -{ - int i, ret, x; - for (i = 0; i < ALPHABET_SIZE*20; i += 1) - { - x = mt_random (&static_mtrand) % (ALPHABET_SIZE/2); - if ((ret = xd3_emit_byte (stream, & data, x))) { return ret; } - } - return 0; -} - -/* Test distribution: An uniform random distribution covering the entire alphabet */ -static int -sec_dist_func7 (xd3_stream *stream, xd3_output *data) -{ - int i, ret, x; - for (i = 0; i < ALPHABET_SIZE*20; i += 1) - { - x = mt_random (&static_mtrand) % ALPHABET_SIZE; - if ((ret = xd3_emit_byte (stream, & data, x))) { return ret; } - } - return 0; -} - -/* Test distribution: A small number of frequent characters, difficult - * to divide into many groups */ -static int -sec_dist_func8 (xd3_stream *stream, xd3_output *data) -{ - int i, ret; - for (i = 0; i < ALPHABET_SIZE*5; i += 1) - { - if ((ret = xd3_emit_byte (stream, & data, 0))) { return ret; } - if ((ret = xd3_emit_byte (stream, & data, 64))) { return ret; } - if ((ret = xd3_emit_byte (stream, & data, 128))) { return ret; } - if ((ret = xd3_emit_byte (stream, & data, 255))) { return ret; } - } - return 0; -} - -/* Test distribution: One that causes many FGK block promotions (found a bug) */ -static int -sec_dist_func9 (xd3_stream *stream, xd3_output *data) -{ - int i, ret; - - int ramp = 0; - int rcount = 0; - int prom = 0; - int pcount = 0; - - /* 200 was long enough to trigger it--only when stricter checking - * that counted all blocks was turned on, but it seems I deleted - * this code. (missing fgk_free_block on line 398). */ - for (i = 0; i < ALPHABET_SIZE*200; i += 1) - { - repeat: - if (ramp < ALPHABET_SIZE) - { - /* Initially Nth symbol has (N+1) frequency */ - if (rcount <= ramp) - { - rcount += 1; - if ((ret = xd3_emit_byte (stream, & data, ramp))) { return ret; } - continue; - } - - ramp += 1; - rcount = 0; - goto repeat; - } - - /* Thereafter, promote least freq to max freq */ - if (pcount == ALPHABET_SIZE) - { - pcount = 0; - prom = (prom + 1) % ALPHABET_SIZE; - } - - pcount += 1; - if ((ret = xd3_emit_byte (stream, & data, prom))) { return ret; } - } - - return 0; -} - -/* Test distribution: freq[i] == i*i, creates a 21-bit code length, fixed in 3.0r. */ -static int -sec_dist_func10 (xd3_stream *stream, xd3_output *data) -{ - int i, j, ret; - for (i = 0; i < ALPHABET_SIZE; i += 1) - { - for (j = 0; j <= (i*i); j += 1) - { - if ((ret = xd3_emit_byte (stream, & data, i))) { return ret; } - } - } - return 0; -} - -/* Test distribution: fibonacci */ -static int -sec_dist_func11 (xd3_stream *stream, xd3_output *data) -{ - int sum0 = 0; - int sum1 = 1; - int i, j, ret; - for (i = 0; i < 33; ++i) - { - for (j = 0; j < (sum0 + sum1); ++j) - { - if ((ret = xd3_emit_byte (stream, & data, i))) { return ret; } - } - sum0 = sum1; - sum1 = j; - } - return 0; -} - -static int -test_secondary_decode (xd3_stream *stream, - const xd3_sec_type *sec, - usize_t input_size, - usize_t compress_size, - const uint8_t *dec_input, - const uint8_t *dec_correct, - uint8_t *dec_output) -{ - int ret; - xd3_sec_stream *dec_stream; - const uint8_t *dec_input_used, *dec_input_end; - uint8_t *dec_output_used, *dec_output_end; - - if ((dec_stream = sec->alloc (stream)) == NULL) { return ENOMEM; } - - sec->init (dec_stream); - - dec_input_used = dec_input; - dec_input_end = dec_input + compress_size; - - dec_output_used = dec_output; - dec_output_end = dec_output + input_size; - - if ((ret = sec->decode (stream, dec_stream, - & dec_input_used, dec_input_end, - & dec_output_used, dec_output_end))) - { - goto fail; - } - - if (dec_input_used != dec_input_end) - { - stream->msg = "unused input"; - ret = XD3_INTERNAL; - goto fail; - } - - if (dec_output_used != dec_output_end) - { - stream->msg = "unfinished output"; - ret = XD3_INTERNAL; - goto fail; - } - - if (memcmp (dec_output, dec_correct, input_size) != 0) - { - stream->msg = "incorrect output"; - ret = XD3_INTERNAL; - goto fail; - } - - fail: - sec->destroy (stream, dec_stream); - return ret; -} - -static int -test_secondary (xd3_stream *stream, const xd3_sec_type *sec, int groups) -{ - int test_i, ret; - xd3_output *in_head, *out_head, *p; - usize_t p_off, input_size, compress_size; - uint8_t *dec_input = NULL, *dec_output = NULL, *dec_correct = NULL; - xd3_sec_stream *enc_stream; - xd3_sec_cfg cfg; - - memset (& cfg, 0, sizeof (cfg)); - - cfg.inefficient = 1; - - for (cfg.ngroups = 1; cfg.ngroups <= groups; cfg.ngroups += 1) - { - DP(RINT "\n..."); - for (test_i = 0; test_i < SIZEOF_ARRAY (sec_dists); test_i += 1) - { - mt_init (& static_mtrand, 0x9f73f7fc); - - in_head = xd3_alloc_output (stream, NULL); - out_head = xd3_alloc_output (stream, NULL); - enc_stream = sec->alloc (stream); - dec_input = NULL; - dec_output = NULL; - dec_correct = NULL; - - if (in_head == NULL || out_head == NULL || enc_stream == NULL) - { - goto nomem; - } - - if ((ret = sec_dists[test_i] (stream, in_head))) { goto fail; } - - sec->init (enc_stream); - - /* Encode data */ - if ((ret = sec->encode (stream, enc_stream, - in_head, out_head, & cfg))) - { - DP(RINT "test %u: encode: %s", test_i, stream->msg); - goto fail; - } - - /* Calculate sizes, allocate contiguous arrays for decoding */ - input_size = xd3_sizeof_output (in_head); - compress_size = xd3_sizeof_output (out_head); - - DP(RINT "%.3f", 8.0 * (double) compress_size / (double) input_size); - - if ((dec_input = xd3_alloc (stream, compress_size, 1)) == NULL || - (dec_output = xd3_alloc (stream, input_size, 1)) == NULL || - (dec_correct = xd3_alloc (stream, input_size, 1)) == NULL) - { - goto nomem; - } - - /* Fill the compressed data array */ - for (p_off = 0, p = out_head; p != NULL; - p_off += p->next, p = p->next_page) - { - memcpy (dec_input + p_off, p->base, p->next); - } - - CHECK(p_off == compress_size); - - /* Fill the input data array */ - for (p_off = 0, p = in_head; p != NULL; - p_off += p->next, p = p->next_page) - { - memcpy (dec_correct + p_off, p->base, p->next); - } - - CHECK(p_off == input_size); - - if ((ret = test_secondary_decode (stream, sec, input_size, - compress_size, dec_input, - dec_correct, dec_output))) - { - DP(RINT "test %u: decode: %s", test_i, stream->msg); - goto fail; - } - - /* Single-bit error test, only cover the first 10 bytes. - * Some non-failures are expected in the Huffman case: - * Changing the clclen array, for example, may not harm the - * decoding. Really looking for faults here. */ - { - int i; - int bytes = min (compress_size, 10U); - for (i = 0; i < bytes * 8; i += 1) - { - dec_input[i/8] ^= 1 << (i%8); - - if ((ret = test_secondary_decode (stream, sec, input_size, - compress_size, dec_input, - dec_correct, dec_output)) - == 0) - { - /*DP(RINT "test %u: decode single-bit [%u/%u] - error non-failure", test_i, i/8, i%8);*/ - } - - dec_input[i/8] ^= 1 << (i%8); - - if ((i % (2*bytes)) == (2*bytes)-1) - { - DOT (); - } - } - ret = 0; - } - - if (0) { nomem: ret = ENOMEM; } - - fail: - sec->destroy (stream, enc_stream); - xd3_free_output (stream, in_head); - xd3_free_output (stream, out_head); - xd3_free (stream, dec_input); - xd3_free (stream, dec_output); - xd3_free (stream, dec_correct); - - if (ret != 0) { return ret; } - } - } - - return 0; -} - -IF_FGK (static int test_secondary_fgk (xd3_stream *stream, int gp) - { return test_secondary (stream, & fgk_sec_type, gp); }) -IF_DJW (static int test_secondary_huff (xd3_stream *stream, int gp) - { return test_secondary (stream, & djw_sec_type, gp); }) -#endif - -/*********************************************************************** - TEST INSTRUCTION TABLE - ***********************************************************************/ - -/* Test that xd3_choose_instruction() does the right thing for its code - * table. */ -static int -test_choose_instruction (xd3_stream *stream, int ignore) -{ - int i; - - stream->code_table = (*stream->code_table_func) (); - - for (i = 0; i < 256; i += 1) - { - const xd3_dinst *d = stream->code_table + i; - xd3_rinst prev, inst; - - CHECK(d->type1 > 0); - - memset (& prev, 0, sizeof (prev)); - memset (& inst, 0, sizeof (inst)); - - if (d->type2 == 0) - { - inst.type = d->type1; - - if ((inst.size = d->size1) == 0) - { - inst.size = TESTBUFSIZE; - } - - XD3_CHOOSE_INSTRUCTION (stream, NULL, & inst); - - if (inst.code2 != 0 || inst.code1 != i) - { - stream->msg = "wrong single instruction"; - return XD3_INTERNAL; - } - } - else - { - prev.type = d->type1; - prev.size = d->size1; - inst.type = d->type2; - inst.size = d->size2; - - XD3_CHOOSE_INSTRUCTION (stream, & prev, & inst); - - if (prev.code2 != i) - { - stream->msg = "wrong double instruction"; - return XD3_INTERNAL; - } - } - } - - return 0; -} - -/*********************************************************************** - TEST INSTRUCTION TABLE CODING - ***********************************************************************/ - -#if GENERIC_ENCODE_TABLES -/* Test that encoding and decoding a code table works */ -static int -test_encode_code_table (xd3_stream *stream, int ignore) -{ - int ret; - const uint8_t *comp_data; - usize_t comp_size; - - if ((ret = xd3_compute_alternate_table_encoding (stream, & comp_data, & comp_size))) - { - return ret; - } - - stream->acache.s_near = __alternate_code_table_desc.near_modes; - stream->acache.s_same = __alternate_code_table_desc.same_modes; - - if ((ret = xd3_apply_table_encoding (stream, comp_data, comp_size))) - { - return ret; - } - - if (memcmp (stream->code_table, xd3_alternate_code_table (), sizeof (xd3_dinst) * 256) != 0) - { - stream->msg = "wrong code table reconstruction"; - return XD3_INTERNAL; - } - - return 0; -} -#endif - -/*********************************************************************** - 64BIT STREAMING - ***********************************************************************/ - -/* This test encodes and decodes a series of 1 megabyte windows, each - * containing a long run of zeros along with a single xoff_t size - * record to indicate the sequence. */ -static int -test_streaming (xd3_stream *in_stream, uint8_t *encbuf, uint8_t *decbuf, uint8_t *delbuf, usize_t megs) -{ - xd3_stream estream, dstream; - int ret; - usize_t i, delsize, decsize; - - if ((ret = xd3_config_stream (& estream, NULL)) || - (ret = xd3_config_stream (& dstream, NULL))) - { - goto fail; - } - - for (i = 0; i < megs; i += 1) - { - ((usize_t*) encbuf)[0] = i; - - if ((i % 200) == 199) { DOT (); } - - if ((ret = xd3_process_stream (1, & estream, xd3_encode_input, 0, - encbuf, 1 << 20, - delbuf, & delsize, 1 << 10))) - { - in_stream->msg = estream.msg; - goto fail; - } - - if ((ret = xd3_process_stream (0, & dstream, xd3_decode_input, 0, - delbuf, delsize, - decbuf, & decsize, 1 << 20))) - { - in_stream->msg = dstream.msg; - goto fail; - } - - if (decsize != 1 << 20 || - memcmp (encbuf, decbuf, 1 << 20) != 0) - { - in_stream->msg = "wrong result"; - ret = XD3_INTERNAL; - goto fail; - } - } - - if ((ret = xd3_close_stream (& estream)) || - (ret = xd3_close_stream (& dstream))) - { - goto fail; - } - - fail: - xd3_free_stream (& estream); - xd3_free_stream (& dstream); - return ret; -} - -/* Run tests of data streaming of over and around 4GB of data. */ -static int -test_compressed_stream_overflow (xd3_stream *stream, int ignore) -{ - int ret; - uint8_t *buf; - - if ((buf = (uint8_t*) malloc (TWO_MEGS_AND_DELTA)) == NULL) { return ENOMEM; } - - memset (buf, 0, TWO_MEGS_AND_DELTA); - - /* Test overflow of a 32-bit file offset. */ - if (SIZEOF_XOFF_T == 4) - { - ret = test_streaming (stream, buf, buf + (1 << 20), buf + (2 << 20), (1 << 12) + 1); - - if (ret == XD3_INVALID_INPUT && MSG_IS ("decoder file offset overflow")) - { - ret = 0; - } - else - { - XPR(NT XD3_LIB_ERRMSG (stream, ret)); - stream->msg = "expected overflow condition"; - ret = XD3_INTERNAL; - goto fail; - } - } - - /* Test transfer of exactly 32bits worth of data. */ - if ((ret = test_streaming (stream, buf, buf + (1 << 20), buf + (2 << 20), 1 << 12))) { goto fail; } - - fail: - free (buf); - return ret; -} - -/*********************************************************************** - COMMAND LINE - ***********************************************************************/ - -/* For each pair of command templates in the array below, test that - * encoding and decoding commands work. Also check for the expected - * size delta, which should be approximately TEST_ADD_RATIO times the - * file size created by test_make_inputs. Due to differences in the - * application header, it is suppressed (-A) so that all delta files - * are the same. */ -static int -test_command_line_arguments (xd3_stream *stream, int ignore) -{ - int i, ret; - - static const char* cmdpairs[] = - { - /* standard input, output */ - "%s %s -A < %s > %s", "%s -d < %s > %s", - "%s %s -A -e < %s > %s", "%s -d < %s > %s", - "%s %s -A= encode < %s > %s", "%s decode < %s > %s", - "%s %s -A -q encode < %s > %s", "%s -qdq < %s > %s", - - /* file input, standard output */ - "%s %s -A= %s > %s", "%s -d %s > %s", - "%s %s -A -e %s > %s", "%s -d %s > %s", - "%s %s encode -A= %s > %s", "%s decode %s > %s", - - /* file input, output */ - "%s %s -A= %s %s", "%s -d %s %s", - "%s %s -A -e %s %s", "%s -d %s %s", - "%s %s -A= encode %s %s", "%s decode %s %s", - - /* option placement */ - "%s %s -A -f %s %s", "%s -f -d %s %s", - "%s %s -e -A= %s %s", "%s -d -f %s %s", - "%s %s -f encode -A= %s %s", "%s -f decode -f %s %s", - }; - - char ecmd[TESTBUFSIZE], dcmd[TESTBUFSIZE]; - int pairs = SIZEOF_ARRAY (cmdpairs) / 2; - xoff_t tsize; - xoff_t dsize; - double ratio; - - mt_init (& static_mtrand, 0x9f73f7fc); - - for (i = 0; i < pairs; i += 1) - { - test_setup (); - if ((ret = test_make_inputs (stream, NULL, & tsize))) { return ret; } - - sprintf (ecmd, cmdpairs[2*i], program_name, - test_softcfg_str, TEST_TARGET_FILE, TEST_DELTA_FILE); - sprintf (dcmd, cmdpairs[2*i+1], program_name, - TEST_DELTA_FILE, TEST_RECON_FILE); - - /* Encode and decode. */ - if ((ret = system (ecmd)) != 0) - { - DP(RINT "xdelta3: encode command: %s\n", ecmd); - stream->msg = "encode cmd failed"; - return XD3_INTERNAL; - } - - if ((ret = system (dcmd)) != 0) - { - DP(RINT "xdelta3: decode command: %s\n", dcmd); - stream->msg = "decode cmd failed"; - return XD3_INTERNAL; - } - - /* Compare the target file. */ - if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) - { - return ret; - } - - if ((ret = test_file_size (TEST_DELTA_FILE, & dsize))) - { - return ret; - } - - ratio = (double) dsize / (double) tsize; - - /* Check that it is not too small, not too large. */ - if (ratio >= TEST_ADD_RATIO + TEST_EPSILON) - { - DP(RINT "xdelta3: test encode with size ratio %.4f, " - "expected < %.4f (%"Q"u, %"Q"u)\n", - ratio, TEST_ADD_RATIO + TEST_EPSILON, dsize, tsize); - stream->msg = "strange encoding"; - return XD3_INTERNAL; - } - - if (ratio <= TEST_ADD_RATIO * (1.0 - 2 * TEST_EPSILON)) - { - DP(RINT "xdelta3: test encode with size ratio %.4f, " - "expected > %.4f\n", - ratio, TEST_ADD_RATIO - TEST_EPSILON); - stream->msg = "strange encoding"; - return XD3_INTERNAL; - } - - /* Also check that compare_files works. The delta and original should - * not be identical. */ - if ((ret = compare_files (stream, TEST_DELTA_FILE, - TEST_TARGET_FILE)) == 0) - { - stream->msg = "broken compare_files"; - return XD3_INTERNAL; - } - - test_cleanup (); - DOT (); - } - - return 0; -} - -static int -check_vcdiff_header (xd3_stream *stream, - const char *input, - const char *line_start, - const char *matches, - int yes_or_no) -{ - int ret; - char vcmd[TESTBUFSIZE], gcmd[TESTBUFSIZE]; - - sprintf (vcmd, "%s printhdr -f %s %s", - program_name, input, TEST_RECON2_FILE); - - if ((ret = system (vcmd)) != 0) - { - DP(RINT "xdelta3: printhdr command: %s\n", vcmd); - stream->msg = "printhdr cmd failed"; - return XD3_INTERNAL; - } - - sprintf (gcmd, "grep \"%s.*%s.*\" %s > /dev/null", - line_start, matches, TEST_RECON2_FILE); - - if (yes_or_no) - { - if ((ret = do_cmd (stream, gcmd))) - { - DP(RINT "xdelta3: %s\n", gcmd); - return ret; - } - } - else - { - if ((ret = do_fail (stream, gcmd))) - { - DP(RINT "xdelta3: %s\n", gcmd); - return ret; - } - } - - return 0; -} - -static int -test_recode_command2 (xd3_stream *stream, int has_source, - int variant, int change) -{ - int has_adler32 = (variant & 0x1) != 0; - int has_apphead = (variant & 0x2) != 0; - int has_secondary = (variant & 0x4) != 0; - - int change_adler32 = (change & 0x1) != 0; - int change_apphead = (change & 0x2) != 0; - int change_secondary = (change & 0x4) != 0; - - int recoded_adler32 = change_adler32 ? !has_adler32 : has_adler32; - int recoded_apphead = change_apphead ? !has_apphead : has_apphead; - int recoded_secondary = change_secondary ? !has_secondary : has_secondary; - - char ecmd[TESTBUFSIZE], recmd[TESTBUFSIZE], dcmd[TESTBUFSIZE]; - xoff_t tsize, ssize; - int ret; - - test_setup (); - - if ((ret = test_make_inputs (stream, has_source ? & ssize : NULL, & tsize))) - { - return ret; - } - - /* First encode */ - sprintf (ecmd, "%s %s -f ", program_name, test_softcfg_str); - strcat (ecmd, has_adler32 ? "" : "-n "); - strcat (ecmd, has_apphead ? "-A=encode_apphead " : "-A= "); - strcat (ecmd, has_secondary ? "-S djw " : "-S none "); - - if (has_source) - { - strcat (ecmd, "-s "); - strcat (ecmd, TEST_SOURCE_FILE); - strcat (ecmd, " "); - } - - strcat (ecmd, TEST_TARGET_FILE); - strcat (ecmd, " "); - strcat (ecmd, TEST_DELTA_FILE); - - if ((ret = system (ecmd)) != 0) - { - DP(RINT "xdelta3: encode command: %s\n", ecmd); - stream->msg = "encode cmd failed"; - return XD3_INTERNAL; - } - - /* Now recode */ - sprintf (recmd, "%s recode %s -f ", program_name, test_softcfg_str); - strcat (recmd, recoded_adler32 ? "" : "-n "); - strcat (recmd, !change_apphead ? "" : - (recoded_apphead ? "-A=recode_apphead " : "-A= ")); - strcat (recmd, recoded_secondary ? "-S djw " : "-S none "); - strcat (recmd, TEST_DELTA_FILE); - strcat (recmd, " "); - strcat (recmd, TEST_COPY_FILE); - - if ((ret = system (recmd)) != 0) - { - DP(RINT "xdelta3: recode command: %s\n", recmd); - stream->msg = "recode cmd failed"; - return XD3_INTERNAL; - } - - /* Check recode changes. */ - if ((ret = check_vcdiff_header (stream, - TEST_COPY_FILE, - "VCDIFF window indicator", - "VCD_SOURCE", - has_source))) { return ret; } - - if ((ret = check_vcdiff_header (stream, - TEST_COPY_FILE, - "VCDIFF header indicator", - "VCD_SECONDARY", - recoded_secondary))) { return ret; } - - if ((ret = check_vcdiff_header (stream, - TEST_COPY_FILE, - "VCDIFF window indicator", - "VCD_ADLER32", - /* Recode can't generate an adler32 - * checksum, it can only preserve it or - * remove it. */ - has_adler32 && recoded_adler32))) - { - return ret; - } - - if (!change_apphead) - { - if ((ret = check_vcdiff_header (stream, - TEST_COPY_FILE, - "VCDIFF header indicator", - "VCD_APPHEADER", - has_apphead))) - { - return ret; - } - if ((ret = check_vcdiff_header (stream, - TEST_COPY_FILE, - "VCDIFF application header", - "encode_apphead", - has_apphead))) - { - return ret; - } - } - else - { - if ((ret = check_vcdiff_header (stream, - TEST_COPY_FILE, - "VCDIFF header indicator", - "VCD_APPHEADER", - recoded_apphead))) - { - return ret; - } - if (recoded_apphead && - (ret = check_vcdiff_header (stream, - TEST_COPY_FILE, - "VCDIFF application header", - "recode_apphead", - 1))) - { - return ret; - } - } - - /* Now decode */ - sprintf (dcmd, "%s -fd ", program_name); - - if (has_source) - { - strcat (dcmd, "-s "); - strcat (dcmd, TEST_SOURCE_FILE); - strcat (dcmd, " "); - } - - strcat (dcmd, TEST_COPY_FILE); - strcat (dcmd, " "); - strcat (dcmd, TEST_RECON_FILE); - - if ((ret = system (dcmd)) != 0) - { - DP(RINT "xdelta3: decode command: %s\n", dcmd); - stream->msg = "decode cmd failed"; - return XD3_INTERNAL; - } - - /* Now compare. */ - if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) - { - return ret; - } - - return 0; -} - -static int -test_recode_command (xd3_stream *stream, int ignore) -{ - /* Things to test: - * - with and without a source file (recode does not change) - * - * (recode may or may not change -- 8 variations) - * - with and without adler32 - * - with and without app header - * - with and without secondary - */ - int has_source; - int variant; - int change; - int ret; - - for (has_source = 0; has_source < 2; has_source++) - { - for (variant = 0; variant < 8; variant++) - { - for (change = 0; change < 8; change++) - { - if ((ret = test_recode_command2 (stream, has_source, - variant, change))) - { - return ret; - } - } - DOT (); - } - } - - return 0; -} - -/*********************************************************************** - EXTERNAL I/O DECOMPRESSION/RECOMPRESSION - ***********************************************************************/ - -#if EXTERNAL_COMPRESSION -/* This performs one step of the test_externally_compressed_io - * function described below. It builds a pipe containing both Xdelta - * and external compression/decompression that should not modify the - * data passing through. */ -static int -test_compressed_pipe (xd3_stream *stream, main_extcomp *ext, char* buf, - const char* comp_options, const char* decomp_options, - int do_ext_recomp, const char* msg) -{ - int ret; - char decomp_buf[TESTBUFSIZE]; - - if (do_ext_recomp) - { - sprintf (decomp_buf, " | %s %s", ext->decomp_cmdname, ext->decomp_options); - } - else - { - decomp_buf[0] = 0; - } - - sprintf (buf, "%s %s < %s | %s %s | %s %s%s > %s", - ext->recomp_cmdname, ext->recomp_options, - TEST_TARGET_FILE, - program_name, comp_options, - program_name, decomp_options, - decomp_buf, - TEST_RECON_FILE); - - if ((ret = system (buf)) != 0) - { - stream->msg = msg; - return XD3_INTERNAL; - } - - if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) - { - return XD3_INTERNAL; - } - - DOT (); - return 0; -} - -/* We want to test that a pipe such as: - * - * --> | gzip -cf | xdelta3 -cf | xdelta3 -dcf | gzip -dcf | --> - * - * is transparent, i.e., does not modify the stream of data. However, - * we also want to verify that at the center the data is properly - * compressed, i.e., that we do not just have a re-compressed gzip - * format, that we have an VCDIFF format. We do this in two steps. - * First test the above pipe, then test with suppressed output - * recompression (-D). The result should be the original input: - * - * --> | gzip -cf | xdelta3 -cf | xdelta3 -Ddcf | --> - * - * Finally we want to test that -D also disables input decompression: - * - * --> | gzip -cf | xdelta3 -Dcf | xdelta3 -Ddcf | gzip -dcf | --> - */ -static int -test_externally_compressed_io (xd3_stream *stream, int ignore) -{ - usize_t i; - int ret; - char buf[TESTBUFSIZE]; - - mt_init (& static_mtrand, 0x9f73f7fc); - - if ((ret = test_make_inputs (stream, NULL, NULL))) { return ret; } - - for (i = 0; i < SIZEOF_ARRAY (extcomp_types); i += 1) - { - main_extcomp *ext = & extcomp_types[i]; - - /* Test for the existence of the external command first, if not skip. */ - sprintf (buf, "%s %s < /dev/null > /dev/null", ext->recomp_cmdname, ext->recomp_options); - - if ((ret = system (buf)) != 0) - { - DP(RINT "%s=0", ext->recomp_cmdname); - continue; - } - - if ((ret = test_compressed_pipe (stream, ext, buf, "-cfq", "-dcfq", 1, - "compression failed: identity pipe")) || - (ret = test_compressed_pipe (stream, ext, buf, "-cfq", "-Rdcfq", 0, - "compression failed: without recompression")) || - (ret = test_compressed_pipe (stream, ext, buf, "-Dcfq", "-Rdcfq", 1, - "compression failed: without decompression"))) - { - return ret; - } - } - - return 0; -} - -/* This tests the proper functioning of external decompression for - * source files. The source and target files are identical and - * compressed by gzip. Decoding such a delta with recompression - * disbaled (-R) should produce the original, uncompressed - * source/target file. Then it checks with output recompression - * enabled--in this case the output should be a compressed copy of the - * original source/target file. Then it checks that encoding with - * decompression disabled works--the compressed files are identical - * and decoding them should always produce a compressed output, - * regardless of -R since the encoded delta file had decompression - * disabled.. - */ -static int -test_source_decompression (xd3_stream *stream, int ignore) -{ - int ret; - char buf[TESTBUFSIZE]; - const main_extcomp *ext; - - mt_init (& static_mtrand, 0x9f73f7fc); - - test_setup (); - if ((ret = test_make_inputs (stream, NULL, NULL))) { return ret; } - - /* Use gzip. */ - if ((ext = main_get_compressor ("G")) == NULL) { DP(RINT "skipped"); return 0; } - - /* Save an uncompressed copy. */ - if ((ret = test_save_copy (TEST_TARGET_FILE))) { return ret; } - - /* Compress the target. */ - sprintf (buf, "%s %s < %s > %s", ext->recomp_cmdname, - ext->recomp_options, TEST_TARGET_FILE, TEST_SOURCE_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Copy back to the source. */ - sprintf (buf, "cp -f %s %s", TEST_SOURCE_FILE, TEST_TARGET_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Now the two identical files are compressed. Delta-encode the target, - * with decompression. */ - sprintf (buf, "%s -eq -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Decode the delta file with recompression disabled, should get an - * uncompressed file out. */ - sprintf (buf, "%s -dq -R -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_COPY_FILE, TEST_RECON_FILE))) { return ret; } - - /* Decode the delta file with recompression, should get a compressed file - * out. But we can't compare compressed files directly. */ - sprintf (buf, "%s -dqf -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - sprintf (buf, "%s %s < %s > %s", ext->decomp_cmdname, ext->decomp_options, TEST_RECON_FILE, TEST_RECON2_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_COPY_FILE, TEST_RECON2_FILE))) { return ret; } - - /* Encode with decompression disabled */ - sprintf (buf, "%s -feqD -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Decode the delta file with recompression enabled, it doesn't matter, - * should get the compressed file out. */ - sprintf (buf, "%s -fdq -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) { return ret; } - - /* Try again with recompression disabled, it doesn't make a difference. */ - sprintf (buf, "%s -fqRd -s%s %s %s", program_name, TEST_SOURCE_FILE, TEST_DELTA_FILE, TEST_RECON_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - if ((ret = compare_files (stream, TEST_TARGET_FILE, TEST_RECON_FILE))) { return ret; } - test_cleanup(); - return 0; -} -#endif - -/*********************************************************************** - FORCE, STDOUT - ***********************************************************************/ - -/* This tests that output will not overwrite an existing file unless - * -f was specified. The test is for encoding (the same code handles - * it for decoding). */ -static int -test_force_behavior (xd3_stream *stream, int ignore) -{ - int ret; - char buf[TESTBUFSIZE]; - - /* Create empty target file */ - test_setup (); - sprintf (buf, "cp /dev/null %s", TEST_TARGET_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Encode to delta file */ - sprintf (buf, "%s -e %s %s", program_name, - TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Encode again, should fail. */ - sprintf (buf, "%s -q -e %s %s ", program_name, - TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_fail (stream, buf))) { return ret; } - - /* Force it, should succeed. */ - sprintf (buf, "%s -f -e %s %s", program_name, - TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - test_cleanup(); - return 0; -} - -/* This checks the proper operation of the -c flag. When specified - * the default output becomes stdout, otherwise the input must be - * provided (encode) or it may be defaulted (decode w/ app header). */ -static int -test_stdout_behavior (xd3_stream *stream, int ignore) -{ - int ret; - char buf[TESTBUFSIZE]; - - test_setup(); - sprintf (buf, "cp /dev/null %s", TEST_TARGET_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Without -c, encode writes to delta file */ - sprintf (buf, "%s -e %s %s", program_name, - TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* With -c, encode writes to stdout */ - sprintf (buf, "%s -e -c %s > %s", program_name, - TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Without -c, decode writes to target file name, but it fails because the - * file exists. */ - sprintf (buf, "%s -q -d %s ", program_name, TEST_DELTA_FILE); - if ((ret = do_fail (stream, buf))) { return ret; } - - /* With -c, decode writes to stdout */ - sprintf (buf, "%s -d -c %s > /dev/null", program_name, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - test_cleanup(); - - return 0; -} - -/* This tests that the no-output flag (-J) works. */ -static int -test_no_output (xd3_stream *stream, int ignore) -{ - int ret; - char buf[TESTBUFSIZE]; - - test_setup (); - - sprintf (buf, "touch %s && chmod 0000 %s", - TEST_NOPERM_FILE, TEST_NOPERM_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - if ((ret = test_make_inputs (stream, NULL, NULL))) { return ret; } - - /* Try no_output encode w/out unwritable output file */ - sprintf (buf, "%s -q -f -e %s %s", program_name, - TEST_TARGET_FILE, TEST_NOPERM_FILE); - if ((ret = do_fail (stream, buf))) { return ret; } - sprintf (buf, "%s -J -e %s %s", program_name, - TEST_TARGET_FILE, TEST_NOPERM_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - /* Now really write the delta to test decode no-output */ - sprintf (buf, "%s -e %s %s", program_name, - TEST_TARGET_FILE, TEST_DELTA_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - - sprintf (buf, "%s -q -f -d %s %s", program_name, - TEST_DELTA_FILE, TEST_NOPERM_FILE); - if ((ret = do_fail (stream, buf))) { return ret; } - sprintf (buf, "%s -J -d %s %s", program_name, - TEST_DELTA_FILE, TEST_NOPERM_FILE); - if ((ret = do_cmd (stream, buf))) { return ret; } - test_cleanup (); - return 0; -} - -/*********************************************************************** - Source identical optimization - ***********************************************************************/ - -/* Computing a delta should be fastest when the two inputs are - * identical, this checks it. The library is called to compute a - * delta between a 10000 byte file, 1000 byte winsize, 500 byte source - * blocksize. The same buffer is used for both source and target. */ -static int -test_identical_behavior (xd3_stream *stream, int ignore) -{ -#define IDB_TGTSZ 10000 -#define IDB_BLKSZ 500 -#define IDB_WINSZ 1000 -#define IDB_DELSZ 1000 -#define IDB_WINCNT (IDB_TGTSZ / IDB_WINSZ) - - int ret, i; - uint8_t buf[IDB_TGTSZ]; - uint8_t del[IDB_DELSZ]; - uint8_t rec[IDB_TGTSZ]; - xd3_source source; - int nextencwin = 0; - int winstarts = 0, winfinishes = 0; - xoff_t srcwin = -1; - usize_t delpos = 0, recsize; - xd3_config config; - - for (i = 0; i < IDB_TGTSZ; i += 1) { buf[i] = mt_random (&static_mtrand); } - - stream->winsize = IDB_WINSZ; - - source.size = IDB_TGTSZ; - source.blksize = IDB_BLKSZ; - source.name = ""; - source.curblk = NULL; - source.curblkno = -1; - - if ((ret = xd3_set_source (stream, & source))) { goto fail; } - - /* Compute an delta between identical source and targets. */ - for (;;) - { - ret = xd3_encode_input (stream); - - if (ret == XD3_INPUT) - { - xd3_avail_input (stream, buf + (IDB_WINSZ * nextencwin), IDB_WINSZ); - nextencwin += 1; - continue; - } - - if (ret == XD3_GETSRCBLK) - { - source.curblkno = source.getblkno; - source.onblk = IDB_BLKSZ; - source.curblk = buf + source.getblkno * IDB_BLKSZ; - srcwin = source.getblkno; - continue; - } - - if (ret == XD3_WINSTART) - { - winstarts++; - continue; - } - if (ret == XD3_WINFINISH) - { - winfinishes++; - if (winfinishes == IDB_WINCNT) - { - break; - } - continue; - } - - if (ret != XD3_OUTPUT) { goto fail; } - - CHECK(delpos + stream->avail_out <= IDB_DELSZ); - - memcpy (del + delpos, stream->next_out, stream->avail_out); - - delpos += stream->avail_out; - - xd3_consume_output (stream); - } - - CHECK(srcwin == source.blocks - 1); - CHECK(winfinishes == IDB_WINCNT); - CHECK(winstarts == IDB_WINCNT); - CHECK(nextencwin == IDB_WINCNT); - - /* Reset. */ - source.blksize = IDB_TGTSZ; - source.onblk = IDB_TGTSZ; - source.curblk = buf; - source.curblkno = 0; - - if ((ret = xd3_close_stream (stream))) { goto fail; } - xd3_free_stream (stream); - xd3_init_config (& config, 0); - if ((ret = xd3_config_stream (stream, & config))) { goto fail; } - if ((ret = xd3_set_source (stream, & source))) { goto fail; } - - /* Decode. */ - if ((ret = xd3_decode_stream (stream, del, delpos, rec, & recsize, IDB_TGTSZ))) { goto fail; } - - /* Check result size and data. */ - if (recsize != IDB_TGTSZ) { stream->msg = "wrong size reconstruction"; goto fail; } - if (memcmp (rec, buf, IDB_TGTSZ) != 0) { stream->msg = "wrong data reconstruction"; goto fail; } - - /* Check that there was one copy per window. */ - IF_DEBUG (if (stream->n_scpy != IDB_WINCNT || - stream->n_add != 0 || - stream->n_run != 0) { stream->msg = "wrong copy count"; goto fail; }); - - /* Check that no checksums were computed because the initial match - was presumed. */ - IF_DEBUG (if (stream->large_ckcnt != 0) { stream->msg = "wrong checksum behavior"; goto fail; }); - - ret = 0; - fail: - return ret; -} - -/*********************************************************************** - String matching test - ***********************************************************************/ - -/* Check particular matching behaviors by calling - * xd3_string_match_soft directly with specific arguments. */ -typedef struct _string_match_test string_match_test; - -typedef enum -{ - SM_NONE = 0, - SM_LAZY = (1 << 1), -} string_match_flags; - -struct _string_match_test -{ - const char *input; - int flags; - const char *result; -}; - -static const string_match_test match_tests[] = -{ - /* nothing */ - { "1234567890", SM_NONE, "" }, - - /* basic run, copy */ - { "11111111112323232323", SM_NONE, "R0/10 C12/8@10" }, - - /* no run smaller than MIN_RUN=8 */ - { "1111111", SM_NONE, "C1/6@0" }, - { "11111111", SM_NONE, "R0/8" }, - - /* simple promotion: the third copy address depends on promotion */ - { "ABCDEF_ABCDEF^ABCDEF", SM_NONE, "C7/6@0 C14/6@7" }, - /* { "ABCDEF_ABCDEF^ABCDEF", SM_PROMOTE, "C7/6@0 C14/6@0" }, forgotten */ - - /* simple lazy: there is a better copy starting with "23 X" than "123 " */ - { "123 23 XYZ 123 XYZ", SM_NONE, "C11/4@0" }, - { "123 23 XYZ 123 XYZ", SM_LAZY, "C11/4@0 C12/6@4" }, - - /* trylazy: no lazy matches unless there are at least two characters beyond - * the first match */ - { "2123_121212", SM_LAZY, "C7/4@5" }, - { "2123_1212123", SM_LAZY, "C7/4@5" }, - { "2123_1212123_", SM_LAZY, "C7/4@5 C8/5@0" }, - - /* trylazy: no lazy matches if the copy is >= MAXLAZY=10 */ - { "2123_121212123_", SM_LAZY, "C7/6@5 C10/5@0" }, - { "2123_12121212123_", SM_LAZY, "C7/8@5 C12/5@0" }, - { "2123_1212121212123_", SM_LAZY, "C7/10@5" }, - - /* lazy run: check a run overlapped by a longer copy */ - { "11111112 111111112 1", SM_LAZY, "C1/6@0 R9/8 C10/10@0" }, - - /* lazy match: match_length,run_l >= min_match tests, shouldn't get any - * copies within the run, no run within the copy */ - { "^________^________ ", SM_LAZY, "R1/8 C9/9@0" }, - - /* chain depth: it only goes back 10. this checks that the 10th match hits - * and the 11th misses. */ - { "1234 1234_1234-1234=1234+1234[1234]1234{1234}1234<1234 ", SM_NONE, - "C5/4@0 C10/4@5 C15/4@10 C20/4@15 C25/4@20 C30/4@25 C35/4@30 C40/4@35 C45/4@40 C50/5@0" }, - { "1234 1234_1234-1234=1234+1234[1234]1234{1234}1234<1234>1234 ", SM_NONE, - "C5/4@0 C10/4@5 C15/4@10 C20/4@15 C25/4@20 C30/4@25 C35/4@30 C40/4@35 C45/4@40 C50/4@45 C55/4@50" }, - - /* ssmatch test */ - { "ABCDE___ABCDE*** BCDE***", SM_NONE, "C8/5@0 C17/4@1" }, - /*{ "ABCDE___ABCDE*** BCDE***", SM_SSMATCH, "C8/5@0 C17/7@9" }, forgotten */ -}; - -static int -test_string_matching (xd3_stream *stream, int ignore) -{ - usize_t i; - int ret; - xd3_config config; - char rbuf[TESTBUFSIZE]; - - for (i = 0; i < SIZEOF_ARRAY (match_tests); i += 1) - { - const string_match_test *test = & match_tests[i]; - char *rptr = rbuf; - usize_t len = strlen (test->input); - - xd3_free_stream (stream); - xd3_init_config (& config, 0); - - config.smatch_cfg = XD3_SMATCH_SOFT; - config.smatcher_soft.large_look = 4; - config.smatcher_soft.large_step = 4; - config.smatcher_soft.small_look = 4; - config.smatcher_soft.small_chain = 10; - config.smatcher_soft.small_lchain = 10; - config.smatcher_soft.max_lazy = (test->flags & SM_LAZY) ? 10 : 0; - config.smatcher_soft.long_enough = 10; - - if ((ret = xd3_config_stream (stream, & config))) { return ret; } - if ((ret = xd3_encode_init_full (stream))) { return ret; } - - xd3_avail_input (stream, (uint8_t*)test->input, len); - - if ((ret = stream->smatcher.string_match (stream))) { return ret; } - - *rptr = 0; - while (! xd3_rlist_empty (& stream->iopt_used)) - { - xd3_rinst *inst = xd3_rlist_pop_front (& stream->iopt_used); - - switch (inst->type) - { - case XD3_RUN: *rptr++ = 'R'; break; - case XD3_CPY: *rptr++ = 'C'; break; - default: CHECK(0); - } - - sprintf (rptr, "%d/%d", inst->pos, inst->size); - rptr += strlen (rptr); - - if (inst->type == XD3_CPY) - { - *rptr++ = '@'; - sprintf (rptr, "%"Q"d", inst->addr); - rptr += strlen (rptr); - } - - *rptr++ = ' '; - - xd3_rlist_push_back (& stream->iopt_free, inst); - } - - if (rptr != rbuf) - { - rptr -= 1; *rptr = 0; - } - - if (strcmp (rbuf, test->result) != 0) - { - DP(RINT "test %u: expected %s: got %s", i, test->result, rbuf); - stream->msg = "wrong result"; - return XD3_INTERNAL; - } - } - - return 0; -} - -/* - * This is a test for many overlapping instructions. It must be a lazy - * matcher. - */ -static int -test_iopt_flush_instructions (xd3_stream *stream, int ignore) -{ - int ret, i; - usize_t tpos = 0; - usize_t delta_size, recon_size; - xd3_config config; - uint8_t target[TESTBUFSIZE]; - uint8_t delta[TESTBUFSIZE]; - uint8_t recon[TESTBUFSIZE]; - - xd3_free_stream (stream); - xd3_init_config (& config, 0); - - config.smatch_cfg = XD3_SMATCH_SOFT; - config.smatcher_soft.large_look = 16; - config.smatcher_soft.large_step = 16; - config.smatcher_soft.small_look = 4; - config.smatcher_soft.small_chain = 128; - config.smatcher_soft.small_lchain = 16; - config.smatcher_soft.max_lazy = 8; - config.smatcher_soft.long_enough = 128; - - if ((ret = xd3_config_stream (stream, & config))) { return ret; } - - for (i = 1; i < 250; i++) - { - target[tpos++] = i; - target[tpos++] = i+1; - target[tpos++] = i+2; - target[tpos++] = i+3; - target[tpos++] = 0; - } - for (i = 1; i < 253; i++) - { - target[tpos++] = i; - } - - if ((ret = xd3_encode_stream (stream, target, tpos, - delta, & delta_size, sizeof (delta)))) - { - return ret; - } - - xd3_free_stream(stream); - if ((ret = xd3_config_stream (stream, & config))) { return ret; } - - if ((ret = xd3_decode_stream (stream, delta, delta_size, - recon, & recon_size, sizeof (recon)))) - { - return ret; - } - - CHECK(tpos == recon_size); - CHECK(memcmp(target, recon, recon_size) == 0); - - return 0; -} - -/* - * This tests the 32/64bit ambiguity for source-window matching. - */ -static int -test_source_cksum_offset (xd3_stream *stream, int ignore) -{ - xd3_source source; - - // Inputs are: - struct { - xoff_t cpos; // stream->srcwin_cksum_pos; - xoff_t ipos; // stream->total_in; - xoff_t size; // stream->src->size; - - usize_t input; // input 32-bit offset - xoff_t output; // output 64-bit offset - - } cksum_test[] = { - // If cpos is <= 2^32 - { 1, 1, 1, 1, 1 }, - -#if XD3_USE_LARGEFILE64 -// cpos ipos size input output -// 0x____xxxxxULL, 0x____xxxxxULL, 0x____xxxxxULL, 0x___xxxxxUL, 0x____xxxxxULL - { 0x100100000ULL, 0x100000000ULL, 0x100200000ULL, 0x00000000UL, 0x100000000ULL }, - { 0x100100000ULL, 0x100000000ULL, 0x100200000ULL, 0xF0000000UL, 0x0F0000000ULL }, - - { 0x100200000ULL, 0x100100000ULL, 0x100200000ULL, 0x00300000UL, 0x000300000ULL }, - - { 25771983104ULL, 25770000000ULL, 26414808769ULL, 2139216707UL, 23614053187ULL }, - -#endif - - { 0, 0, 0, 0, 0 }, - }, *test_ptr; - - stream->src = &source; - - for (test_ptr = cksum_test; test_ptr->cpos; test_ptr++) { - xoff_t r; - stream->srcwin_cksum_pos = test_ptr->cpos; - stream->total_in = test_ptr->ipos; - stream->src->size = test_ptr->size; - - r = xd3_source_cksum_offset(stream, test_ptr->input); - CHECK(r == test_ptr->output); - } - return 0; -} - -static int -test_in_memory (xd3_stream *stream, int ignore) -{ - // test_text is 256 bytes - uint8_t ibuf[sizeof(test_text)]; - uint8_t dbuf[sizeof(test_text)]; - uint8_t obuf[sizeof(test_text)]; - usize_t size = sizeof(test_text); - usize_t dsize, osize; - int r1, r2; - int eflags = SECONDARY_DJW ? XD3_SEC_DJW : 0; - - memcpy(ibuf, test_text, size); - memset(ibuf + 128, 0, 16); - - r1 = xd3_encode_memory(ibuf, size, - test_text, size, - dbuf, &dsize, size, eflags); - - r2 = xd3_decode_memory(dbuf, dsize, - test_text, size, - obuf, &osize, size, 0); - - if (r1 != 0 || r2 != 0 || dsize >= (size/2) || dsize < 1 || - osize != size) { - stream->msg = "encode/decode size error"; - return XD3_INTERNAL; - } - - if (memcmp(obuf, ibuf, size) != 0) { - stream->msg = "encode/decode data error"; - return XD3_INTERNAL; - } - - return 0; -} - -/*********************************************************************** - TEST MAIN - ***********************************************************************/ - -static int -xd3_selftest (void) -{ -#define DO_TEST(fn,flags,arg) \ - do { \ - xd3_stream stream; \ - xd3_config config; \ - xd3_init_config (& config, flags); \ - DP(RINT "xdelta3: testing " #fn "%s...", \ - flags ? (" (" #flags ")") : ""); \ - if ((ret = xd3_config_stream (& stream, & config) == 0) && \ - (ret = test_ ## fn (& stream, arg)) == 0) { \ - DP(RINT " success\n"); \ - } else { \ - DP(RINT " failed: %s: %s\n", xd3_errstring (& stream), \ - xd3_mainerror (ret)); } \ - xd3_free_stream (& stream); \ - if (ret != 0) { goto failure; } \ - } while (0) - - int ret; - -#ifndef WIN32 - if (getuid() == 0) - { - DP(RINT "xdelta3: This test should not be run as root.\n"); - ret = XD3_INVALID; - goto failure; - } -#endif - - DO_TEST (random_numbers, 0, 0); - - DO_TEST (decode_integer_end_of_input, 0, 0); - DO_TEST (decode_integer_overflow, 0, 0); - DO_TEST (encode_decode_uint32_t, 0, 0); - DO_TEST (encode_decode_uint64_t, 0, 0); - DO_TEST (usize_t_overflow, 0, 0); - DO_TEST (forward_match, 0, 0); - - DO_TEST (address_cache, 0, 0); - IF_GENCODETBL (DO_TEST (address_cache, XD3_ALT_CODE_TABLE, 0)); - - DO_TEST (string_matching, 0, 0); - DO_TEST (choose_instruction, 0, 0); - DO_TEST (identical_behavior, 0, 0); - DO_TEST (in_memory, 0, 0); - - IF_GENCODETBL (DO_TEST (choose_instruction, XD3_ALT_CODE_TABLE, 0)); - IF_GENCODETBL (DO_TEST (encode_code_table, 0, 0)); - - DO_TEST (iopt_flush_instructions, 0, 0); - DO_TEST (source_cksum_offset, 0, 0); - - DO_TEST (decompress_single_bit_error, 0, 3); - DO_TEST (decompress_single_bit_error, XD3_ADLER32, 3); - - IF_FGK (DO_TEST (decompress_single_bit_error, XD3_SEC_FGK, 3)); - IF_DJW (DO_TEST (decompress_single_bit_error, XD3_SEC_DJW, 8)); - - /* There are many expected non-failures for ALT_CODE_TABLE because - * not all of the instruction codes are used. */ - IF_GENCODETBL ( - DO_TEST (decompress_single_bit_error, XD3_ALT_CODE_TABLE, 224)); - -#ifndef WIN32 - DO_TEST (force_behavior, 0, 0); - DO_TEST (stdout_behavior, 0, 0); - DO_TEST (no_output, 0, 0); - DO_TEST (command_line_arguments, 0, 0); - DO_TEST (recode_command, 0, 0); - -#if EXTERNAL_COMPRESSION - DO_TEST (source_decompression, 0, 0); - DO_TEST (externally_compressed_io, 0, 0); -#endif - -#endif /* WIN32 */ - - IF_DJW (DO_TEST (secondary_huff, 0, DJW_MAX_GROUPS)); - IF_FGK (DO_TEST (secondary_fgk, 0, 1)); - - DO_TEST (compressed_stream_overflow, 0, 0); - -failure: - test_cleanup (); - return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE; -#undef DO_TEST -} diff --git a/xdelta3-test.py b/xdelta3-test.py deleted file mode 100755 index 7fcc3d5..0000000 --- a/xdelta3-test.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/python2.5 -# xdelta 3 - delta compression tools and library -# Copyright (C) 2003, 2006, 2007. Joshua P. MacDonald -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import xdelta3 - -# the test data section is expected to be len('target') -source = 'source source input0 source source' -target = 'source source target source source' - -# -# - -print 'encode: basic ...' -result, patch = xdelta3.xd3_encode_memory(target, source, 50) - -assert result == 0 -assert len(patch) < len(source) - -print 'encode: adler32 ...' -result, patch_adler32 = xdelta3.xd3_encode_memory(target, source, 50, - xdelta3.XD3_ADLER32) - -assert result == 0 -assert len(patch_adler32) < len(source) -assert len(patch_adler32) > len(patch) - -print 'encode: secondary ...' -result, patch_djw = xdelta3.xd3_encode_memory(target, source, 50, - xdelta3.XD3_SEC_DJW) - -assert result == 0 -# secondary compression doesn't help -assert len(patch_djw) > len(patch) - -print 'encode: exact ...' -result, ignore = xdelta3.xd3_encode_memory(target, source, len(patch)) - -assert result == 0 -assert len(ignore) < len(source) - -print 'encode: out of space ...' -result, ignore = xdelta3.xd3_encode_memory(target, source, len(patch) - 1) - -assert result == 28 -assert ignore == None - -print 'encode: zero space ...' -result, ignore = xdelta3.xd3_encode_memory(target, source, 0) - -assert result == 28 -assert ignore == None - -print 'encode: no source ...' -result, zdata = xdelta3.xd3_encode_memory(target, None, 50) - -assert result == 0 -assert len(zdata) > len(patch) - -print 'encode: no input ...' -result, ignore = xdelta3.xd3_encode_memory(None, None, 50) - -assert result != 0 - -print 'decode: basic ...' -result, target1 = xdelta3.xd3_decode_memory(patch, source, len(target)) - -assert result == 0 -assert len(target1) == len(target) -assert target1 == target - -print 'decode: out of space ...' -result, ignore = xdelta3.xd3_decode_memory(patch, source, len(target) - 1) - -assert result == 28 -assert ignore == None - -print 'decode: zero space ...' -result, ignore = xdelta3.xd3_decode_memory(patch, source, 0) - -assert result == 28 -assert ignore == None - -print 'decode: single byte error ...' -# a few expected single-byte errors, e.g., unused address cache bits, see -# xdelta3-test.h's single-bit error tests -extra_count = 4 -noverify_count = 0 -for corrupt_pos in range(len(patch_adler32)): - input = ''.join([j == corrupt_pos and '\xff' or patch_adler32[j] - for j in range(len(patch_adler32))]) - - result, ignore = xdelta3.xd3_decode_memory(input, source, len(target), 0) - assert result == -17712 - assert ignore == None - - # without adler32 verification, the error may be in the data section which - # in this case is 6 bytes 'target' - result, corrupt = xdelta3.xd3_decode_memory(input, source, len(target), - xdelta3.XD3_ADLER32_NOVER) - if result == 0: - noverify_count = noverify_count + 1 - #print "got %s" % corrupt - #end -#end -assert noverify_count == len('target') + extra_count - -print 'decode: no source ...' -result, target2 = xdelta3.xd3_decode_memory(zdata, None, len(target)) - -assert result == 0 -assert target == target2 - -# Test compression level setting via flags. assumes a 9 byte checksum -# and that level 9 steps 2, level 1 steps 15: -# 01234567890123456789012345678901 -# level 1 only indexes 2 checksums "abcdefghi" and "ABCDEFGHI" -# outputs 43 vs. 23 bytes -print 'encode: compression level ...' - -source = '_la_la_abcdefghi_la_la_ABCDEFGHI' -target = 'la_la_ABCDEFGH__la_la_abcdefgh__' - -result1, level1 = xdelta3.xd3_encode_memory(target, source, 50, xdelta3.XD3_COMPLEVEL_1) -result9, level9 = xdelta3.xd3_encode_memory(target, source, 50, xdelta3.XD3_COMPLEVEL_9) - -assert result1 == 0 and result9 == 0 -assert len(level1) > len(level9) - -# -# Issue 65 -print 'encode: 65 ...' -source = 'Hello World' -target = 'Hello everyone' -result, patch = xdelta3.xd3_encode_memory(target, source, len(target)) -assert result != 0 - -result, patch = xdelta3.xd3_encode_memory(target, source, 2 * len(target)) -assert result == 0 - -print 'PASS' diff --git a/xdelta3.c b/xdelta3.c deleted file mode 100644 index 9028d0c..0000000 --- a/xdelta3.c +++ /dev/null @@ -1,5337 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - ------------------------------------------------------------------- - - Xdelta 3 - - The goal of this library is to to implement both the (stand-alone) - data-compression and delta-compression aspects of VCDIFF encoding, and - to support a programming interface that works like Zlib - (http://www.gzip.org/zlib.html). See RFC3284: The VCDIFF Generic - Differencing and Compression Data Format. - - VCDIFF is a unified encoding that combines data-compression and - delta-encoding ("differencing"). - - VCDIFF has a detailed byte-code instruction set with many features. - The instruction format supports an immediate size operand for small - COPYs and ADDs (e.g., under 18 bytes). There are also instruction - "modes", which are used to compress COPY addresses by using two - address caches. An instruction mode refers to slots in the NEAR - and SAME caches for recent addresses. NEAR remembers the - previous 4 (by default) COPY addresses, and SAME catches - frequent re-uses of the same address using a 3-way (by default) - 256-entry associative cache of [ADDR mod 256], the encoded byte. - A hit in the NEAR/SAME cache requires 0/1 ADDR bytes. - - VCDIFF has a default instruction table, but an alternate - instruction tables may themselves be be delta-compressed and - included in the encoding header. This allows even more freedom. - There are 9 instruction modes in the default code table, 4 near, 3 - same, VCD_SELF (absolute encoding) and VCD_HERE (relative to the - current position). - - ---------------------------------------------------------------------- - - Algorithms - - Aside from the details of encoding and decoding, there are a bunch - of algorithms needed. - - 1. STRING-MATCH. A two-level fingerprinting approach is used. A - single loop computes the two checksums -- small and large -- at - successive offsets in the TARGET file. The large checksum is more - accurate and is used to discover SOURCE matches, which are - potentially very long. The small checksum is used to discover - copies within the TARGET. Small matching, which is more expensive, - usually dominates the large STRING-MATCH costs in this code - the - more exhaustive the search, the better the results. Either of the - two string-matching mechanisms may be disabled. - - 2. INSTRUCTION SELECTION. The IOPT buffer here represents a queue - used to store overlapping copy instructions. There are two possible - optimizations that go beyond a greedy search. Both of these fall - into the category of "non-greedy matching" optimizations. - - The first optimization stems from backward SOURCE-COPY matching. - When a new SOURCE-COPY instruction covers a previous instruction in - the target completely, it is erased from the queue. Randal Burns - originally analyzed these algorithms and did a lot of related work - (\cite the 1.5-pass algorithm). - - The second optimization comes by the encoding of common very-small - COPY and ADD instructions, for which there are special DOUBLE-code - instructions, which code two instructions in a single byte. - - The cost of bad instruction-selection overhead is relatively high - for data-compression, relative to delta-compression, so this second - optimization is fairly important. With "lazy" matching (the name - used in Zlib for a similar optimization), the string-match - algorithm searches after a match for potential overlapping copy - instructions. In Xdelta and by default, VCDIFF, the minimum match - size is 4 bytes, whereas Zlib searches with a 3-byte minimum. This - feature, combined with double instructions, provides a nice - challenge. Search in this file for "black magic", a heuristic. - - 3. STREAM ALIGNMENT. Stream alignment is needed to compress large - inputs in constant space. See xd3_srcwin_move_point(). - - 4. WINDOW SELECTION. When the IOPT buffer flushes, in the first call - to xd3_iopt_finish_encoding containing any kind of copy instruction, - the parameters of the source window must be decided: the offset into - the source and the length of the window. Since the IOPT buffer is - finite, the program may be forced to fix these values before knowing - the best offset/length. - - 5. SECONDARY COMPRESSION. VCDIFF supports a secondary encoding to - be applied to the individual sections of the data format, which are - ADDRess, INSTruction, and DATA. Several secondary compressor - variations are implemented here, although none is standardized yet. - - One is an adaptive huffman algorithm -- the FGK algorithm (Faller, - Gallager, and Knuth, 1985). This compressor is extremely slow. - - The other is a simple static Huffman routine, which is the base - case of a semi-adaptive scheme published by D.J. Wheeler and first - widely used in bzip2 (by Julian Seward). This is a very - interesting algorithm, originally published in nearly cryptic form - by D.J. Wheeler. !!!NOTE!!! Because these are not standardized, - secondary compression remains off by default. - ftp://ftp.cl.cam.ac.uk/users/djw3/bred3.{c,ps} - -------------------------------------------------------------------- - - Other Features - - 1. USER CONVENIENCE - - For user convenience, it is essential to recognize Gzip-compressed - files and automatically Gzip-decompress them prior to - delta-compression (or else no delta-compression will be achieved - unless the user manually decompresses the inputs). The compressed - represention competes with Xdelta, and this must be hidden from the - command-line user interface. The Xdelta-1.x encoding was simple, not - compressed itself, so Xdelta-1.x uses Zlib internally to compress the - representation. - - This implementation supports external compression, which implements - the necessary fork() and pipe() mechanics. There is a tricky step - involved to support automatic detection of a compressed input in a - non-seekable input. First you read a bit of the input to detect - magic headers. When a compressed format is recognized, exec() the - external compression program and create a second child process to - copy the original input stream. [Footnote: There is a difficulty - related to using Gzip externally. It is not possible to decompress - and recompress a Gzip file transparently. If FILE.GZ had a - cryptographic signature, then, after: (1) Gzip-decompression, (2) - Xdelta-encoding, (3) Gzip-compression the signature could be - broken. The only way to solve this problem is to guess at Gzip's - compression level or control it by other means. I recommend that - specific implementations of any compression scheme store - information needed to exactly re-compress the input, that way - external compression is transparent - however, this won't happen - here until it has stabilized.] - - 2. APPLICATION-HEADER - - This feature was introduced in RFC3284. It allows any application - to include a header within the VCDIFF file format. This allows - general inter-application data exchange with support for - application-specific extensions to communicate metadata. - - 3. VCDIFF CHECKSUM - - An optional checksum value is included with each window, which can - be used to validate the final result. This verifies the correct source - file was used for decompression as well as the obvious advantage: - checking the implementation (and underlying) correctness. - - 4. LIGHT WEIGHT - - The code makes efforts to avoid copying data more than necessary. - The code delays many initialization tasks until the first use, it - optimizes for identical (perfectly matching) inputs. It does not - compute any checksums until the first lookup misses. Memory usage - is reduced. String-matching is templatized (by slightly gross use - of CPP) to hard-code alternative compile-time defaults. The code - has few outside dependencies. - ---------------------------------------------------------------------- - - The default rfc3284 instruction table: - (see RFC for the explanation) - - TYPE SIZE MODE TYPE SIZE MODE INDEX - -------------------------------------------------------------------- - 1. Run 0 0 Noop 0 0 0 - 2. Add 0, [1,17] 0 Noop 0 0 [1,18] - 3. Copy 0, [4,18] 0 Noop 0 0 [19,34] - 4. Copy 0, [4,18] 1 Noop 0 0 [35,50] - 5. Copy 0, [4,18] 2 Noop 0 0 [51,66] - 6. Copy 0, [4,18] 3 Noop 0 0 [67,82] - 7. Copy 0, [4,18] 4 Noop 0 0 [83,98] - 8. Copy 0, [4,18] 5 Noop 0 0 [99,114] - 9. Copy 0, [4,18] 6 Noop 0 0 [115,130] - 10. Copy 0, [4,18] 7 Noop 0 0 [131,146] - 11. Copy 0, [4,18] 8 Noop 0 0 [147,162] - 12. Add [1,4] 0 Copy [4,6] 0 [163,174] - 13. Add [1,4] 0 Copy [4,6] 1 [175,186] - 14. Add [1,4] 0 Copy [4,6] 2 [187,198] - 15. Add [1,4] 0 Copy [4,6] 3 [199,210] - 16. Add [1,4] 0 Copy [4,6] 4 [211,222] - 17. Add [1,4] 0 Copy [4,6] 5 [223,234] - 18. Add [1,4] 0 Copy 4 6 [235,238] - 19. Add [1,4] 0 Copy 4 7 [239,242] - 20. Add [1,4] 0 Copy 4 8 [243,246] - 21. Copy 4 [0,8] Add 1 0 [247,255] - -------------------------------------------------------------------- - - Reading the source: Overview - - This file includes itself in several passes to macro-expand certain - sections with variable forms. Just read ahead, there's only a - little confusion. I know this sounds ugly, but hard-coding some of - the string-matching parameters results in a 10-15% increase in - string-match performance. The only time this hurts is when you have - unbalanced #if/endifs. - - A single compilation unit tames the Makefile. In short, this is to - allow the above-described hack without an explodingMakefile. The - single compilation unit includes the core library features, - configurable string-match templates, optional main() command-line - tool, misc optional features, and a regression test. Features are - controled with CPP #defines, see Makefile.am. - - The initial __XDELTA3_C_HEADER_PASS__ starts first, the _INLINE_ and - _TEMPLATE_ sections follow. Easy stuff first, hard stuff last. - - Optional features include: - - xdelta3-main.h The command-line interface, external compression - support, POSIX-specific, info & VCDIFF-debug tools. - xdelta3-second.h The common secondary compression routines. - xdelta3-decoder.h All decoding routines. - xdelta3-djw.h The semi-adaptive huffman secondary encoder. - xdelta3-fgk.h The adaptive huffman secondary encoder. - xdelta3-test.h The unit test covers major algorithms, - encoding and decoding. There are single-bit - error decoding tests. There are 32/64-bit file size - boundary tests. There are command-line tests. - There are compression tests. There are external - compression tests. There are string-matching tests. - There should be more tests... - - Additional headers include: - - xdelta3.h The public header file. - xdelta3-cfgs.h The default settings for default, built-in - encoders. These are hard-coded at - compile-time. There is also a single - soft-coded string matcher for experimenting - with arbitrary values. - xdelta3-list.h A cyclic list template - - Misc little debug utilities: - - badcopy.c Randomly modifies an input file based on two - parameters: (1) the probability that a byte in - the file is replaced with a pseudo-random value, - and (2) the mean change size. Changes are - generated using an expoential distribution - which approximates the expected error_prob - distribution. - -------------------------------------------------------------------- - - This file itself is unusually large. I hope to defend this layout - with lots of comments. Everything in this file is related to - encoding and decoding. I like it all together - the template stuff - is just a hack. */ - -#ifndef __XDELTA3_C_HEADER_PASS__ -#define __XDELTA3_C_HEADER_PASS__ - -#include <errno.h> -#include <string.h> - -#include "xdelta3.h" - -/*********************************************************************** - STATIC CONFIGURATION - ***********************************************************************/ - -#ifndef XD3_MAIN /* the main application */ -#define XD3_MAIN 0 -#endif - -#ifndef VCDIFF_TOOLS -#define VCDIFF_TOOLS XD3_MAIN -#endif - -#ifndef SECONDARY_FGK /* one from the algorithm preservation department: */ -#define SECONDARY_FGK 0 /* adaptive Huffman routines */ -#endif - -#ifndef SECONDARY_DJW /* semi-adaptive/static Huffman for the eventual */ -#define SECONDARY_DJW 0 /* standardization, off by default until such time. */ -#endif - -#ifndef GENERIC_ENCODE_TABLES /* These three are the RFC-spec'd app-specific */ -#define GENERIC_ENCODE_TABLES 0 /* code features. This is tested but not recommended */ -#endif /* unless there's a real application. */ -#ifndef GENERIC_ENCODE_TABLES_COMPUTE -#define GENERIC_ENCODE_TABLES_COMPUTE 0 -#endif -#ifndef GENERIC_ENCODE_TABLES_COMPUTE_PRINT -#define GENERIC_ENCODE_TABLES_COMPUTE_PRINT 0 -#endif - -#if XD3_ENCODER -#define IF_ENCODER(x) x -#else -#define IF_ENCODER(x) -#endif - -/***********************************************************************/ - -typedef enum { - - /* header indicator bits */ - VCD_SECONDARY = (1 << 0), /* uses secondary compressor */ - VCD_CODETABLE = (1 << 1), /* supplies code table data */ - VCD_APPHEADER = (1 << 2), /* supplies application data */ - VCD_INVHDR = ~7U, - - /* window indicator bits */ - VCD_SOURCE = (1 << 0), /* copy window in source file */ - VCD_TARGET = (1 << 1), /* copy window in target file */ - VCD_ADLER32 = (1 << 2), /* has adler32 checksum */ - VCD_INVWIN = ~7U, - - VCD_SRCORTGT = VCD_SOURCE | VCD_TARGET, - - /* delta indicator bits */ - VCD_DATACOMP = (1 << 0), - VCD_INSTCOMP = (1 << 1), - VCD_ADDRCOMP = (1 << 2), - VCD_INVDEL = ~0x7U, - -} xd3_indicator; - -typedef enum { - VCD_DJW_ID = 1, - VCD_FGK_ID = 16, /* Note: these are not standard IANA-allocated IDs! */ -} xd3_secondary_ids; - -typedef enum { - SEC_NOFLAGS = 0, - - /* Note: SEC_COUNT_FREQS Not implemented (to eliminate 1st Huffman pass) */ - SEC_COUNT_FREQS = (1 << 0), -} xd3_secondary_flags; - -typedef enum { - DATA_SECTION, /* These indicate which section to the secondary - * compressor. */ - INST_SECTION, /* The header section is not compressed, therefore not - * listed here. */ - ADDR_SECTION, -} xd3_section_type; - -typedef enum -{ - XD3_NOOP = 0, - XD3_ADD = 1, - XD3_RUN = 2, - XD3_CPY = 3, /* XD3_CPY rtypes are represented as (XD3_CPY + - * copy-mode value) */ -} xd3_rtype; - -/***********************************************************************/ - -#include "xdelta3-list.h" - -XD3_MAKELIST(xd3_rlist, xd3_rinst, link); - -/***********************************************************************/ - -#define SECONDARY_MIN_SAVINGS 2 /* Secondary compression has to save - at least this many bytes. */ -#define SECONDARY_MIN_INPUT 10 /* Secondary compression needs at - least this many bytes. */ - -#define VCDIFF_MAGIC1 0xd6 /* 1st file byte */ -#define VCDIFF_MAGIC2 0xc3 /* 2nd file byte */ -#define VCDIFF_MAGIC3 0xc4 /* 3rd file byte */ -#define VCDIFF_VERSION 0x00 /* 4th file byte */ - -#define VCD_SELF 0 /* 1st address mode */ -#define VCD_HERE 1 /* 2nd address mode */ - -#define CODE_TABLE_STRING_SIZE (6 * 256) /* Should fit a code table string. */ -#define CODE_TABLE_VCDIFF_SIZE (6 * 256) /* Should fit a compressed code - * table string */ - -#define SECONDARY_ANY (SECONDARY_DJW || SECONDARY_FGK) - -#define ALPHABET_SIZE 256 /* Used in test code--size of the secondary - * compressor alphabet. */ - -#define HASH_PERMUTE 1 /* The input is permuted by random nums */ -#define ADLER_LARGE_CKSUM 1 /* Adler checksum vs. RK checksum */ - -#define HASH_CKOFFSET 1U /* Table entries distinguish "no-entry" from - * offset 0 using this offset. */ - -#define MIN_SMALL_LOOK 2U /* Match-optimization stuff. */ -#define MIN_LARGE_LOOK 2U -#define MIN_MATCH_OFFSET 1U -#define MAX_MATCH_SPLIT 18U /* VCDIFF code table: 18 is the default limit - * for direct-coded ADD sizes */ - -#define LEAST_MATCH_INCR 0 /* The least number of bytes an overlapping - * match must beat the preceding match by. This - * is a bias for the lazy match optimization. A - * non-zero value means that an adjacent match - * has to be better by more than the step - * between them. 0. */ - -#define MIN_MATCH 4U /* VCDIFF code table: MIN_MATCH=4 */ -#define MIN_ADD 1U /* 1 */ -#define MIN_RUN 8U /* The shortest run, if it is shorter than this - * an immediate add/copy will be just as good. - * ADD1/COPY6 = 1I+1D+1A bytes, RUN18 = - * 1I+1D+1A. */ - -#define MAX_MODES 9 /* Maximum number of nodes used for - * compression--does not limit decompression. */ - -#define ENC_SECTS 4 /* Number of separate output sections. */ - -#define HDR_TAIL(s) ((s)->enc_tails[0]) -#define DATA_TAIL(s) ((s)->enc_tails[1]) -#define INST_TAIL(s) ((s)->enc_tails[2]) -#define ADDR_TAIL(s) ((s)->enc_tails[3]) - -#define HDR_HEAD(s) ((s)->enc_heads[0]) -#define DATA_HEAD(s) ((s)->enc_heads[1]) -#define INST_HEAD(s) ((s)->enc_heads[2]) -#define ADDR_HEAD(s) ((s)->enc_heads[3]) - -#define SIZEOF_ARRAY(x) (sizeof(x) / sizeof(x[0])) - -#define TOTAL_MODES(x) (2+(x)->acache.s_same+(x)->acache.s_near) - -/* Template instances. */ -#if XD3_BUILD_SLOW -#define IF_BUILD_SLOW(x) x -#else -#define IF_BUILD_SLOW(x) -#endif -#if XD3_BUILD_FAST -#define IF_BUILD_FAST(x) x -#else -#define IF_BUILD_FAST(x) -#endif -#if XD3_BUILD_FASTER -#define IF_BUILD_FASTER(x) x -#else -#define IF_BUILD_FASTER(x) -#endif -#if XD3_BUILD_FASTEST -#define IF_BUILD_FASTEST(x) x -#else -#define IF_BUILD_FASTEST(x) -#endif -#if XD3_BUILD_SOFT -#define IF_BUILD_SOFT(x) x -#else -#define IF_BUILD_SOFT(x) -#endif -#if XD3_BUILD_DEFAULT -#define IF_BUILD_DEFAULT(x) x -#else -#define IF_BUILD_DEFAULT(x) -#endif - -/* Consume N bytes of input, only used by the decoder. */ -#define DECODE_INPUT(n) \ - do { \ - stream->total_in += (xoff_t) (n); \ - stream->avail_in -= (n); \ - stream->next_in += (n); \ - } while (0) - -/* Update the run-length state */ -#define NEXTRUN(c) do { if ((c) == run_c) { run_l += 1; } \ - else { run_c = (c); run_l = 1; } } while (0) - -/* This CPP-conditional stuff can be cleaned up... */ -#if XD3_DEBUG -#define IF_DEBUG(x) x -#else -#define IF_DEBUG(x) -#endif -#if XD3_DEBUG > 1 -#define IF_DEBUG1(x) x -#else -#define IF_DEBUG1(x) -#endif -#if XD3_DEBUG > 2 -#define IF_DEBUG2(x) x -#else -#define IF_DEBUG2(x) -#endif -#if REGRESSION_TEST -#define IF_REGRESSION(x) x -#else -#define IF_REGRESSION(x) -#endif - -/***********************************************************************/ - -#if XD3_ENCODER -static void* xd3_alloc0 (xd3_stream *stream, - usize_t elts, - usize_t size); - - -static xd3_output* xd3_alloc_output (xd3_stream *stream, - xd3_output *old_output); - -static int xd3_alloc_iopt (xd3_stream *stream, int elts); - -static void xd3_free_output (xd3_stream *stream, - xd3_output *output); - -static int xd3_emit_byte (xd3_stream *stream, - xd3_output **outputp, - uint8_t code); - -static int xd3_emit_bytes (xd3_stream *stream, - xd3_output **outputp, - const uint8_t *base, - usize_t size); - -static int xd3_emit_double (xd3_stream *stream, xd3_rinst *first, - xd3_rinst *second, usize_t code); -static int xd3_emit_single (xd3_stream *stream, xd3_rinst *single, - usize_t code); - -static usize_t xd3_sizeof_output (xd3_output *output); -static void xd3_encode_reset (xd3_stream *stream); - -static int xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos); -static int xd3_source_extend_match (xd3_stream *stream); -static int xd3_srcwin_setup (xd3_stream *stream); -static usize_t xd3_iopt_last_matched (xd3_stream *stream); -static int xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, - uint32_t num); - -static usize_t xd3_smatch (xd3_stream *stream, - usize_t base, - usize_t scksum, - usize_t *match_offset); -static int xd3_string_match_init (xd3_stream *stream); -static uint32_t xd3_scksum (uint32_t *state, const uint8_t *seg, const int ln); -static int xd3_comprun (const uint8_t *seg, int slook, uint8_t *run_cp); -static int xd3_srcwin_move_point (xd3_stream *stream, - usize_t *next_move_point); - -static int xd3_emit_run (xd3_stream *stream, usize_t pos, - usize_t size, uint8_t run_c); -static usize_t xd3_checksum_hash (const xd3_hash_cfg *cfg, - const usize_t cksum); -static xoff_t xd3_source_cksum_offset(xd3_stream *stream, usize_t low); -static void xd3_scksum_insert (xd3_stream *stream, - usize_t inx, - usize_t scksum, - usize_t pos); - - -#if XD3_DEBUG -static void xd3_verify_run_state (xd3_stream *stream, - const uint8_t *inp, - int x_run_l, - uint8_t x_run_c); -static void xd3_verify_large_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum); -static void xd3_verify_small_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum); - -#endif /* XD3_DEBUG */ -#endif /* XD3_ENCODER */ - -static int xd3_decode_allocate (xd3_stream *stream, usize_t size, - uint8_t **copied1, usize_t *alloc1); - -static void xd3_compute_code_table_string (const xd3_dinst *code_table, - uint8_t *str); -static void* xd3_alloc (xd3_stream *stream, usize_t elts, usize_t size); -static void xd3_free (xd3_stream *stream, void *ptr); - -static int xd3_read_uint32_t (xd3_stream *stream, const uint8_t **inpp, - const uint8_t *max, uint32_t *valp); - -#if REGRESSION_TEST -static int xd3_selftest (void); -#endif - -/***********************************************************************/ - -#define UINT32_OFLOW_MASK 0xfe000000U -#define UINT64_OFLOW_MASK 0xfe00000000000000ULL - -#ifndef UINT32_MAX -#define UINT32_MAX 4294967295U -#endif - -#ifndef UINT64_MAX -#define UINT64_MAX 18446744073709551615ULL -#endif - -#if SIZEOF_USIZE_T == 4 -#define USIZE_T_MAX UINT32_MAX -#define xd3_decode_size xd3_decode_uint32_t -#define xd3_emit_size xd3_emit_uint32_t -#define xd3_sizeof_size xd3_sizeof_uint32_t -#define xd3_read_size xd3_read_uint32_t -#elif SIZEOF_USIZE_T == 8 -#define USIZE_T_MAX UINT64_MAX -#define xd3_decode_size xd3_decode_uint64_t -#define xd3_emit_size xd3_emit_uint64_t -#define xd3_sizeof_size xd3_sizeof_uint64_t -#define xd3_read_size xd3_read_uint64_t -#endif - -#if SIZEOF_XOFF_T == 4 -#define XOFF_T_MAX UINT32_MAX -#define xd3_decode_offset xd3_decode_uint32_t -#define xd3_emit_offset xd3_emit_uint32_t -#elif SIZEOF_XOFF_T == 8 -#define XOFF_T_MAX UINT64_MAX -#define xd3_decode_offset xd3_decode_uint64_t -#define xd3_emit_offset xd3_emit_uint64_t -#endif - -#define USIZE_T_OVERFLOW(a,b) ((USIZE_T_MAX - (usize_t) (a)) < (usize_t) (b)) -#define XOFF_T_OVERFLOW(a,b) ((XOFF_T_MAX - (xoff_t) (a)) < (xoff_t) (b)) - -const char* xd3_strerror (int ret) -{ - switch (ret) - { - case XD3_INPUT: return "XD3_INPUT"; - case XD3_OUTPUT: return "XD3_OUTPUT"; - case XD3_GETSRCBLK: return "XD3_GETSRCBLK"; - case XD3_GOTHEADER: return "XD3_GOTHEADER"; - case XD3_WINSTART: return "XD3_WINSTART"; - case XD3_WINFINISH: return "XD3_WINFINISH"; - case XD3_TOOFARBACK: return "XD3_TOOFARBACK"; - case XD3_INTERNAL: return "XD3_INTERNAL"; - case XD3_INVALID_INPUT: return "XD3_INVALID_INPUT"; - } - return NULL; -} - -/***********************************************************************/ - -#define xd3_sec_data(s) ((s)->sec_stream_d) -#define xd3_sec_inst(s) ((s)->sec_stream_i) -#define xd3_sec_addr(s) ((s)->sec_stream_a) - -struct _xd3_sec_type -{ - int id; - const char *name; - xd3_secondary_flags flags; - - /* xd3_sec_stream is opaque to the generic code */ - xd3_sec_stream* (*alloc) (xd3_stream *stream); - void (*destroy) (xd3_stream *stream, - xd3_sec_stream *sec); - void (*init) (xd3_sec_stream *sec); - int (*decode) (xd3_stream *stream, - xd3_sec_stream *sec_stream, - const uint8_t **input, - const uint8_t *input_end, - uint8_t **output, - const uint8_t *output_end); -#if XD3_ENCODER - int (*encode) (xd3_stream *stream, - xd3_sec_stream *sec_stream, - xd3_output *input, - xd3_output *output, - xd3_sec_cfg *cfg); -#endif -}; - -#define BIT_STATE_ENCODE_INIT { 0, 1 } -#define BIT_STATE_DECODE_INIT { 0, 0x100 } - -typedef struct _bit_state bit_state; -struct _bit_state -{ - usize_t cur_byte; - usize_t cur_mask; -}; - -#if SECONDARY_ANY == 0 -#define IF_SEC(x) -#define IF_NSEC(x) x -#else /* yuck */ -#define IF_SEC(x) x -#define IF_NSEC(x) -static int -xd3_decode_secondary (xd3_stream *stream, - xd3_desect *sect, - xd3_sec_stream **sec_streamp); -#if XD3_ENCODER -static int -xd3_encode_secondary (xd3_stream *stream, - xd3_output **head, - xd3_output **tail, - xd3_sec_stream **sec_streamp, - xd3_sec_cfg *cfg, - int *did_it); -#endif -#endif /* SECONDARY_ANY */ - -#if SECONDARY_FGK -static const xd3_sec_type fgk_sec_type; -#define IF_FGK(x) x -#define FGK_CASE(s) \ - s->sec_type = & fgk_sec_type; \ - break; -#else -#define IF_FGK(x) -#define FGK_CASE(s) \ - s->msg = "unavailable secondary compressor: FGK Adaptive Huffman"; \ - return XD3_INTERNAL; -#endif - -#if SECONDARY_DJW -static const xd3_sec_type djw_sec_type; -#define IF_DJW(x) x -#define DJW_CASE(s) \ - s->sec_type = & djw_sec_type; \ - break; -#else -#define IF_DJW(x) -#define DJW_CASE(s) \ - s->msg = "unavailable secondary compressor: DJW Static Huffman"; \ - return XD3_INTERNAL; -#endif - -/***********************************************************************/ - -#include "xdelta3-hash.h" - -/* Process template passes - this includes xdelta3.c several times. */ -#define __XDELTA3_C_TEMPLATE_PASS__ -#include "xdelta3-cfgs.h" -#undef __XDELTA3_C_TEMPLATE_PASS__ - -/* Process the inline pass. */ -#define __XDELTA3_C_INLINE_PASS__ -#include "xdelta3.c" -#undef __XDELTA3_C_INLINE_PASS__ - -/* Secondary compression */ -#if SECONDARY_ANY -#include "xdelta3-second.h" -#endif - -#if SECONDARY_FGK -#include "xdelta3-fgk.h" -static const xd3_sec_type fgk_sec_type = -{ - VCD_FGK_ID, - "FGK Adaptive Huffman", - SEC_NOFLAGS, - (xd3_sec_stream* (*)()) fgk_alloc, - (void (*)()) fgk_destroy, - (void (*)()) fgk_init, - (int (*)()) xd3_decode_fgk, - IF_ENCODER((int (*)()) xd3_encode_fgk) -}; -#endif - -#if SECONDARY_DJW -#include "xdelta3-djw.h" -static const xd3_sec_type djw_sec_type = -{ - VCD_DJW_ID, - "Static Huffman", - SEC_COUNT_FREQS, - (xd3_sec_stream* (*)()) djw_alloc, - (void (*)()) djw_destroy, - (void (*)()) djw_init, - (int (*)()) xd3_decode_huff, - IF_ENCODER((int (*)()) xd3_encode_huff) -}; -#endif - -#if XD3_MAIN || PYTHON_MODULE || SWIG_MODULE || NOT_MAIN -#include "xdelta3-main.h" -#endif - -#if REGRESSION_TEST -#include "xdelta3-test.h" -#endif - -#if PYTHON_MODULE -#include "xdelta3-python.h" -#endif - -#endif /* __XDELTA3_C_HEADER_PASS__ */ -#ifdef __XDELTA3_C_INLINE_PASS__ - -/**************************************************************** - Instruction tables - *****************************************************************/ - -/* The following code implements a parametrized description of the - * code table given above for a few reasons. It is not necessary for - * implementing the standard, to support compression with variable - * tables, so an implementation is only required to know the default - * code table to begin decompression. (If the encoder uses an - * alternate table, the table is included in compressed form inside - * the VCDIFF file.) - * - * Before adding variable-table support there were two functions which - * were hard-coded to the default table above. - * xd3_compute_default_table() would create the default table by - * filling a 256-elt array of xd3_dinst values. The corresponding - * function, xd3_choose_instruction(), would choose an instruction - * based on the hard-coded parameters of the default code table. - * - * Notes: The parametrized code table description here only generates - * tables of a certain regularity similar to the default table by - * allowing to vary the distribution of single- and - * double-instructions and change the number of near and same copy - * modes. More exotic tables are only possible by extending this - * code. - * - * For performance reasons, both the parametrized and non-parametrized - * versions of xd3_choose_instruction remain. The parametrized - * version is only needed for testing multi-table decoding support. - * If ever multi-table encoding is required, this can be optimized by - * compiling static functions for each table. - */ - -/* The XD3_CHOOSE_INSTRUCTION calls xd3_choose_instruction with the - * table description when GENERIC_ENCODE_TABLES are in use. The - * IF_GENCODETBL macro enables generic-code-table specific code. */ -#if GENERIC_ENCODE_TABLES -#define XD3_CHOOSE_INSTRUCTION(stream,prev,inst) xd3_choose_instruction (stream->code_table_desc, prev, inst) -#define IF_GENCODETBL(x) x -#else -#define XD3_CHOOSE_INSTRUCTION(stream,prev,inst) xd3_choose_instruction (prev, inst) -#define IF_GENCODETBL(x) -#endif - -/* This structure maintains information needed by - * xd3_choose_instruction to compute the code for a double instruction - * by first indexing an array of code_table_sizes by copy mode, then - * using (offset + (muliplier * X)) */ -struct _xd3_code_table_sizes { - uint8_t cpy_max; - uint8_t offset; - uint8_t mult; -}; - -/* This contains a complete description of a code table. */ -struct _xd3_code_table_desc -{ - /* Assumes a single RUN instruction */ - /* Assumes that MIN_MATCH is 4 */ - - uint8_t add_sizes; /* Number of immediate-size single adds (default 17) */ - uint8_t near_modes; /* Number of near copy modes (default 4) */ - uint8_t same_modes; /* Number of same copy modes (default 3) */ - uint8_t cpy_sizes; /* Number of immediate-size single copies (default 15) */ - - uint8_t addcopy_add_max; /* Maximum add size for an add-copy double instruction, - all modes (default 4) */ - uint8_t addcopy_near_cpy_max; /* Maximum cpy size for an add-copy double instruction, - up through VCD_NEAR modes (default 6) */ - uint8_t addcopy_same_cpy_max; /* Maximum cpy size for an add-copy double instruction, - VCD_SAME modes (default 4) */ - - uint8_t copyadd_add_max; /* Maximum add size for a copy-add double instruction, - all modes (default 1) */ - uint8_t copyadd_near_cpy_max; /* Maximum cpy size for a copy-add double instruction, - up through VCD_NEAR modes (default 4) */ - uint8_t copyadd_same_cpy_max; /* Maximum cpy size for a copy-add double instruction, - VCD_SAME modes (default 4) */ - - xd3_code_table_sizes addcopy_max_sizes[MAX_MODES]; - xd3_code_table_sizes copyadd_max_sizes[MAX_MODES]; -}; - -/* The rfc3284 code table is represented: */ -static const xd3_code_table_desc __rfc3284_code_table_desc = { - 17, /* add sizes */ - 4, /* near modes */ - 3, /* same modes */ - 15, /* copy sizes */ - - 4, /* add-copy max add */ - 6, /* add-copy max cpy, near */ - 4, /* add-copy max cpy, same */ - - 1, /* copy-add max add */ - 4, /* copy-add max cpy, near */ - 4, /* copy-add max cpy, same */ - - /* addcopy */ - { {6,163,3},{6,175,3},{6,187,3},{6,199,3},{6,211,3},{6,223,3},{4,235,1},{4,239,1},{4,243,1} }, - /* copyadd */ - { {4,247,1},{4,248,1},{4,249,1},{4,250,1},{4,251,1},{4,252,1},{4,253,1},{4,254,1},{4,255,1} }, -}; - -#if GENERIC_ENCODE_TABLES -/* An alternate code table for testing (5 near, 0 same): - * - * TYPE SIZE MODE TYPE SIZE MODE INDEX - * --------------------------------------------------------------- - * 1. Run 0 0 Noop 0 0 0 - * 2. Add 0, [1,23] 0 Noop 0 0 [1,24] - * 3. Copy 0, [4,20] 0 Noop 0 0 [25,42] - * 4. Copy 0, [4,20] 1 Noop 0 0 [43,60] - * 5. Copy 0, [4,20] 2 Noop 0 0 [61,78] - * 6. Copy 0, [4,20] 3 Noop 0 0 [79,96] - * 7. Copy 0, [4,20] 4 Noop 0 0 [97,114] - * 8. Copy 0, [4,20] 5 Noop 0 0 [115,132] - * 9. Copy 0, [4,20] 6 Noop 0 0 [133,150] - * 10. Add [1,4] 0 Copy [4,6] 0 [151,162] - * 11. Add [1,4] 0 Copy [4,6] 1 [163,174] - * 12. Add [1,4] 0 Copy [4,6] 2 [175,186] - * 13. Add [1,4] 0 Copy [4,6] 3 [187,198] - * 14. Add [1,4] 0 Copy [4,6] 4 [199,210] - * 15. Add [1,4] 0 Copy [4,6] 5 [211,222] - * 16. Add [1,4] 0 Copy [4,6] 6 [223,234] - * 17. Copy 4 [0,6] Add [1,3] 0 [235,255] - * --------------------------------------------------------------- */ -static const xd3_code_table_desc __alternate_code_table_desc = { - 23, /* add sizes */ - 5, /* near modes */ - 0, /* same modes */ - 17, /* copy sizes */ - - 4, /* add-copy max add */ - 6, /* add-copy max cpy, near */ - 0, /* add-copy max cpy, same */ - - 3, /* copy-add max add */ - 4, /* copy-add max cpy, near */ - 0, /* copy-add max cpy, same */ - - /* addcopy */ - { {6,151,3},{6,163,3},{6,175,3},{6,187,3},{6,199,3},{6,211,3},{6,223,3},{0,0,0},{0,0,0} }, - /* copyadd */ - { {4,235,1},{4,238,1},{4,241,1},{4,244,1},{4,247,1},{4,250,1},{4,253,1},{0,0,0},{0,0,0} }, -}; -#endif - -/* Computes code table entries of TBL using the specified description. */ -static void -xd3_build_code_table (const xd3_code_table_desc *desc, xd3_dinst *tbl) -{ - usize_t size1, size2, mode; - usize_t cpy_modes = 2 + desc->near_modes + desc->same_modes; - xd3_dinst *d = tbl; - - (d++)->type1 = XD3_RUN; - (d++)->type1 = XD3_ADD; - - for (size1 = 1; size1 <= desc->add_sizes; size1 += 1, d += 1) - { - d->type1 = XD3_ADD; - d->size1 = size1; - } - - for (mode = 0; mode < cpy_modes; mode += 1) - { - (d++)->type1 = XD3_CPY + mode; - - for (size1 = MIN_MATCH; size1 < MIN_MATCH + desc->cpy_sizes; size1 += 1, d += 1) - { - d->type1 = XD3_CPY + mode; - d->size1 = size1; - } - } - - for (mode = 0; mode < cpy_modes; mode += 1) - { - for (size1 = 1; size1 <= desc->addcopy_add_max; size1 += 1) - { - usize_t max = (mode < 2U + desc->near_modes) ? - desc->addcopy_near_cpy_max : - desc->addcopy_same_cpy_max; - - for (size2 = MIN_MATCH; size2 <= max; size2 += 1, d += 1) - { - d->type1 = XD3_ADD; - d->size1 = size1; - d->type2 = XD3_CPY + mode; - d->size2 = size2; - } - } - } - - for (mode = 0; mode < cpy_modes; mode += 1) - { - usize_t max = (mode < 2U + desc->near_modes) ? - desc->copyadd_near_cpy_max : - desc->copyadd_same_cpy_max; - - for (size1 = MIN_MATCH; size1 <= max; size1 += 1) - { - for (size2 = 1; size2 <= desc->copyadd_add_max; size2 += 1, d += 1) - { - d->type1 = XD3_CPY + mode; - d->size1 = size1; - d->type2 = XD3_ADD; - d->size2 = size2; - } - } - } - - XD3_ASSERT (d - tbl == 256); -} - -/* This function generates the static default code table. */ -static const xd3_dinst* -xd3_rfc3284_code_table (void) -{ - static xd3_dinst __rfc3284_code_table[256]; - - if (__rfc3284_code_table[0].type1 != XD3_RUN) - { - xd3_build_code_table (& __rfc3284_code_table_desc, __rfc3284_code_table); - } - - return __rfc3284_code_table; -} - -#if XD3_ENCODER -#if GENERIC_ENCODE_TABLES -/* This function generates the alternate code table. */ -static const xd3_dinst* -xd3_alternate_code_table (void) -{ - static xd3_dinst __alternate_code_table[256]; - - if (__alternate_code_table[0].type1 != XD3_RUN) - { - xd3_build_code_table (& __alternate_code_table_desc, __alternate_code_table); - } - - return __alternate_code_table; -} - -/* This function computes the ideal second instruction INST based on - * preceding instruction PREV. If it is possible to issue a double - * instruction based on this pair it sets PREV->code2, otherwise it - * sets INST->code1. */ -static void -xd3_choose_instruction (const xd3_code_table_desc *desc, xd3_rinst *prev, xd3_rinst *inst) -{ - switch (inst->type) - { - case XD3_RUN: - /* The 0th instruction is RUN */ - inst->code1 = 0; - break; - - case XD3_ADD: - - if (inst->size > desc->add_sizes) - { - /* The first instruction is non-immediate ADD */ - inst->code1 = 1; - } - else - { - /* The following ADD_SIZES instructions are immediate ADDs */ - inst->code1 = 1 + inst->size; - - /* Now check for a possible COPY-ADD double instruction */ - if (prev != NULL) - { - int prev_mode = prev->type - XD3_CPY; - - /* If previous is a copy. Note: as long as the previous - * is not a RUN instruction, it should be a copy because - * it cannot be an add. This check is more clear. */ - if (prev_mode >= 0 && inst->size <= desc->copyadd_add_max) - { - const xd3_code_table_sizes *sizes = & desc->copyadd_max_sizes[prev_mode]; - - /* This check and the inst->size-<= above are == in - the default table. */ - if (prev->size <= sizes->cpy_max) - { - /* The second and third exprs are 0 in the - default table. */ - prev->code2 = sizes->offset + - (sizes->mult * (prev->size - MIN_MATCH)) + - (inst->size - MIN_ADD); - } - } - } - } - break; - - default: - { - int mode = inst->type - XD3_CPY; - - /* The large copy instruction is offset by the run, large add, - * and immediate adds, then multipled by the number of - * immediate copies plus one (the large copy) (i.e., if there - * are 15 immediate copy instructions then there are 16 copy - * instructions per mode). */ - inst->code1 = 2 + desc->add_sizes + (1 + desc->cpy_sizes) * mode; - - /* Now if the copy is short enough for an immediate instruction. */ - if (inst->size < MIN_MATCH + desc->cpy_sizes && - /* TODO: there needs to be a more comprehensive test for this - * boundary condition, merge is now exercising code in which - * size < MIN_MATCH is possible and it's unclear if the above - * size < (MIN_MATCH + cpy_sizes) should be a <= from inspection - * of the default table version below. */ - inst->size >= MIN_MATCH) - { - inst->code1 += inst->size + 1 - MIN_MATCH; - - /* Now check for a possible ADD-COPY double instruction. */ - if ( (prev != NULL) && - (prev->type == XD3_ADD) && - (prev->size <= desc->addcopy_add_max) ) - { - const xd3_code_table_sizes *sizes = & desc->addcopy_max_sizes[mode]; - - if (inst->size <= sizes->cpy_max) - { - prev->code2 = sizes->offset + - (sizes->mult * (prev->size - MIN_ADD)) + - (inst->size - MIN_MATCH); - } - } - } - } - } -} -#else /* GENERIC_ENCODE_TABLES */ - -/* This version of xd3_choose_instruction is hard-coded for the default - table. */ -static void -xd3_choose_instruction (xd3_rinst *prev, xd3_rinst *inst) -{ - switch (inst->type) - { - case XD3_RUN: - inst->code1 = 0; - break; - - case XD3_ADD: - inst->code1 = 1; - - if (inst->size <= 17) - { - inst->code1 += inst->size; - - if ( (inst->size == 1) && - (prev != NULL) && - (prev->size == 4) && - (prev->type >= XD3_CPY) ) - { - prev->code2 = 247 + (prev->type - XD3_CPY); - } - } - - break; - - default: - { - int mode = inst->type - XD3_CPY; - - XD3_ASSERT (inst->type >= XD3_CPY && inst->type < 12); - - inst->code1 = 19 + 16 * mode; - - if (inst->size <= 18 && inst->size >= 4) - { - inst->code1 += inst->size - 3; - - if ( (prev != NULL) && - (prev->type == XD3_ADD) && - (prev->size <= 4) ) - { - if ( (inst->size <= 6) && - (mode <= 5) ) - { - prev->code2 = 163 + (mode * 12) + (3 * (prev->size - 1)) + (inst->size - 4); - - XD3_ASSERT (prev->code2 <= 234); - } - else if ( (inst->size == 4) && - (mode >= 6) ) - { - prev->code2 = 235 + ((mode - 6) * 4) + (prev->size - 1); - - XD3_ASSERT (prev->code2 <= 246); - } - } - } - - XD3_ASSERT (inst->code1 <= 162); - } - break; - } -} -#endif /* GENERIC_ENCODE_TABLES */ - -/*********************************************************************** - Instruction table encoder/decoder - ***********************************************************************/ - -#if GENERIC_ENCODE_TABLES -#if GENERIC_ENCODE_TABLES_COMPUTE == 0 - -/* In this case, we hard-code the result of - * compute_code_table_encoding for each alternate code table, - * presuming that saves time/space. This has been 131 bytes, but - * secondary compression was turned off. */ -static const uint8_t __alternate_code_table_compressed[178] = -{0xd6,0xc3,0xc4,0x00,0x00,0x01,0x8a,0x6f,0x40,0x81,0x27,0x8c,0x00,0x00,0x4a,0x4a,0x0d,0x02,0x01,0x03, -0x01,0x03,0x00,0x01,0x00,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e, -0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x00,0x01,0x01,0x01,0x02,0x02,0x02,0x03,0x03,0x03,0x04, -0x04,0x04,0x04,0x00,0x04,0x05,0x06,0x01,0x02,0x03,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x05,0x05,0x05, -0x06,0x06,0x06,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x00,0x02,0x00,0x18,0x13,0x63,0x00,0x1b,0x00,0x54, -0x00,0x15,0x23,0x6f,0x00,0x28,0x13,0x54,0x00,0x15,0x01,0x1a,0x31,0x23,0x6c,0x0d,0x23,0x48,0x00,0x15, -0x93,0x6f,0x00,0x28,0x04,0x23,0x51,0x04,0x32,0x00,0x2b,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x12,0x00, -0x12,0x00,0x12,0x53,0x57,0x9c,0x07,0x43,0x6f,0x00,0x34,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00, -0x0c,0x00,0x0c,0x00,0x15,0x00,0x82,0x6f,0x00,0x15,0x12,0x0c,0x00,0x03,0x03,0x00,0x06,0x00,}; - -static int -xd3_compute_alternate_table_encoding (xd3_stream *stream, const uint8_t **data, usize_t *size) -{ - (*data) = __alternate_code_table_compressed; - (*size) = sizeof (__alternate_code_table_compressed); - return 0; -} - -#else - -/* The alternate code table will be computed and stored here. */ -static uint8_t __alternate_code_table_compressed[CODE_TABLE_VCDIFF_SIZE]; -static usize_t __alternate_code_table_compressed_size; - -/* This function generates a delta describing the code table for - * encoding within a VCDIFF file. This function is NOT thread safe - * because it is only intended that this function is used to generate - * statically-compiled strings. */ -int xd3_compute_code_table_encoding (xd3_stream *in_stream, - const xd3_dinst *code_table, - uint8_t *comp_string, - usize_t *comp_string_size) -{ - /* TODO: use xd3_encode_memory() */ - uint8_t dflt_string[CODE_TABLE_STRING_SIZE]; - uint8_t code_string[CODE_TABLE_STRING_SIZE]; - xd3_stream stream; - xd3_source source; - xd3_config config; - int ret; - - memset (& source, 0, sizeof (source)); - - xd3_compute_code_table_string (xd3_rfc3284_code_table (), dflt_string); - xd3_compute_code_table_string (code_table, code_string); - - /* Use DJW secondary compression if it is on by default. This saves - * about 20 bytes. */ - xd3_init_config (& config, XD3_FLUSH | (SECONDARY_DJW ? XD3_SEC_DJW : 0)); - - /* Be exhaustive. */ - config.sprevsz = 1<<11; - config.srcwin_maxsz = CODE_TABLE_STRING_SIZE; - - config.smatch_cfg = XD3_SMATCH_SOFT; - config.smatcher_soft.large_look = 4; - config.smatcher_soft.large_step = 1; - config.smatcher_soft.small_look = 4; - config.smatcher_soft.small_chain = CODE_TABLE_STRING_SIZE; - config.smatcher_soft.small_lchain = CODE_TABLE_STRING_SIZE; - config.smatcher_soft.max_lazy = CODE_TABLE_STRING_SIZE; - config.smatcher_soft.long_enough = CODE_TABLE_STRING_SIZE; - - if ((ret = xd3_config_stream (& stream, & config))) { goto fail; } - - source.size = CODE_TABLE_STRING_SIZE; - source.blksize = CODE_TABLE_STRING_SIZE; - source.onblk = CODE_TABLE_STRING_SIZE; - source.name = ""; - source.curblk = dflt_string; - source.curblkno = 0; - - if ((ret = xd3_set_source (& stream, & source))) { goto fail; } - - if ((ret = xd3_encode_stream (& stream, code_string, CODE_TABLE_STRING_SIZE, - comp_string, comp_string_size, CODE_TABLE_VCDIFF_SIZE))) { goto fail; } - - fail: - - in_stream->msg = stream.msg; - xd3_free_stream (& stream); - return ret; -} - -/* Compute a delta between alternate and rfc3284 tables. As soon as - * another alternate table is added, this code should become generic. - * For now there is only one alternate table for testing. */ -static int -xd3_compute_alternate_table_encoding (xd3_stream *stream, const uint8_t **data, usize_t *size) -{ - int ret; - - if (__alternate_code_table_compressed[0] == 0) - { - if ((ret = xd3_compute_code_table_encoding (stream, xd3_alternate_code_table (), - __alternate_code_table_compressed, - & __alternate_code_table_compressed_size))) - { - return ret; - } - - /* During development of a new code table, enable this variable to print - * the new static contents and determine its size. At run time the - * table will be filled in appropriately, but at least it should have - * the proper size beforehand. */ -#if GENERIC_ENCODE_TABLES_COMPUTE_PRINT - { - int i; - - DP(RINT, "\nstatic const usize_t __alternate_code_table_compressed_size = %u;\n", - __alternate_code_table_compressed_size); - - DP(RINT, "static const uint8_t __alternate_code_table_compressed[%u] =\n{", - __alternate_code_table_compressed_size); - - for (i = 0; i < __alternate_code_table_compressed_size; i += 1) - { - DP(RINT, "0x%02x,", __alternate_code_table_compressed[i]); - if ((i % 20) == 19) { DP(RINT, "\n"); } - } - - DP(RINT, "};\n"); - } -#endif - } - - (*data) = __alternate_code_table_compressed; - (*size) = __alternate_code_table_compressed_size; - - return 0; -} -#endif /* GENERIC_ENCODE_TABLES_COMPUTE != 0 */ -#endif /* GENERIC_ENCODE_TABLES */ - -#endif /* XD3_ENCODER */ - -/* This function generates the 1536-byte string specified in sections 5.4 and - * 7 of rfc3284, which is used to represent a code table within a VCDIFF - * file. */ -void xd3_compute_code_table_string (const xd3_dinst *code_table, uint8_t *str) -{ - int i, s; - - XD3_ASSERT (CODE_TABLE_STRING_SIZE == 6 * 256); - - for (s = 0; s < 6; s += 1) - { - for (i = 0; i < 256; i += 1) - { - switch (s) - { - case 0: *str++ = (code_table[i].type1 >= XD3_CPY ? XD3_CPY : code_table[i].type1); break; - case 1: *str++ = (code_table[i].type2 >= XD3_CPY ? XD3_CPY : code_table[i].type2); break; - case 2: *str++ = (code_table[i].size1); break; - case 3: *str++ = (code_table[i].size2); break; - case 4: *str++ = (code_table[i].type1 >= XD3_CPY ? code_table[i].type1 - XD3_CPY : 0); break; - case 5: *str++ = (code_table[i].type2 >= XD3_CPY ? code_table[i].type2 - XD3_CPY : 0); break; - } - } - } -} - -/* This function translates the code table string into the internal representation. The - * stream's near and same-modes should already be set. */ -static int -xd3_apply_table_string (xd3_stream *stream, const uint8_t *code_string) -{ - int i, s; - int modes = TOTAL_MODES (stream); - xd3_dinst *code_table; - - if ((code_table = stream->code_table_alloc = - (xd3_dinst*) xd3_alloc (stream, sizeof (xd3_dinst), 256)) == NULL) - { - return ENOMEM; - } - - for (s = 0; s < 6; s += 1) - { - for (i = 0; i < 256; i += 1) - { - switch (s) - { - case 0: - if (*code_string > XD3_CPY) - { - stream->msg = "invalid code-table opcode"; - return XD3_INTERNAL; - } - code_table[i].type1 = *code_string++; - break; - case 1: - if (*code_string > XD3_CPY) - { - stream->msg = "invalid code-table opcode"; - return XD3_INTERNAL; - } - code_table[i].type2 = *code_string++; - break; - case 2: - if (*code_string != 0 && code_table[i].type1 == XD3_NOOP) - { - stream->msg = "invalid code-table size"; - return XD3_INTERNAL; - } - code_table[i].size1 = *code_string++; - break; - case 3: - if (*code_string != 0 && code_table[i].type2 == XD3_NOOP) - { - stream->msg = "invalid code-table size"; - return XD3_INTERNAL; - } - code_table[i].size2 = *code_string++; - break; - case 4: - if (*code_string >= modes) - { - stream->msg = "invalid code-table mode"; - return XD3_INTERNAL; - } - if (*code_string != 0 && code_table[i].type1 != XD3_CPY) - { - stream->msg = "invalid code-table mode"; - return XD3_INTERNAL; - } - code_table[i].type1 += *code_string++; - break; - case 5: - if (*code_string >= modes) - { - stream->msg = "invalid code-table mode"; - return XD3_INTERNAL; - } - if (*code_string != 0 && code_table[i].type2 != XD3_CPY) - { - stream->msg = "invalid code-table mode"; - return XD3_INTERNAL; - } - code_table[i].type2 += *code_string++; - break; - } - } - } - - stream->code_table = code_table; - return 0; -} - -/* This function applies a code table delta and returns an actual code table. */ -static int -xd3_apply_table_encoding (xd3_stream *in_stream, const uint8_t *data, usize_t size) -{ - uint8_t dflt_string[CODE_TABLE_STRING_SIZE]; - uint8_t code_string[CODE_TABLE_STRING_SIZE]; - usize_t code_size; - xd3_stream stream; - xd3_source source; - int ret; - - /* The default code table string can be cached if alternate code tables ever become - * popular. */ - xd3_compute_code_table_string (xd3_rfc3284_code_table (), dflt_string); - - source.size = CODE_TABLE_STRING_SIZE; - source.blksize = CODE_TABLE_STRING_SIZE; - source.onblk = CODE_TABLE_STRING_SIZE; - source.name = "rfc3284 code table"; - source.curblk = dflt_string; - source.curblkno = 0; - - if ((ret = xd3_config_stream (& stream, NULL)) || - (ret = xd3_set_source (& stream, & source)) || - (ret = xd3_decode_stream (& stream, data, size, code_string, & code_size, sizeof (code_string)))) - { - in_stream->msg = stream.msg; - goto fail; - } - - if (code_size != sizeof (code_string)) - { - stream.msg = "corrupt code-table encoding"; - ret = XD3_INTERNAL; - goto fail; - } - - if ((ret = xd3_apply_table_string (in_stream, code_string))) { goto fail; } - - fail: - - xd3_free_stream (& stream); - return ret; -} - -/***********************************************************************/ - -static inline void -xd3_swap_uint8p (uint8_t** p1, uint8_t** p2) -{ - uint8_t *t = (*p1); - (*p1) = (*p2); - (*p2) = t; -} - -static inline void -xd3_swap_usize_t (usize_t* p1, usize_t* p2) -{ - usize_t t = (*p1); - (*p1) = (*p2); - (*p2) = t; -} - -/* It's not constant time, but it computes the log. */ -static int -xd3_check_pow2 (usize_t value, usize_t *logof) -{ - usize_t x = 1; - usize_t nolog; - if (logof == NULL) { - logof = &nolog; - } - - *logof = 0; - - for (; x != 0; x <<= 1, *logof += 1) - { - if (x == value) - { - return 0; - } - } - - return XD3_INTERNAL; -} - -static usize_t -xd3_pow2_roundup (usize_t x) -{ - usize_t i = 1; - while (x > i) { - i <<= 1; - } - return i; -} - -static usize_t -xd3_round_blksize (usize_t sz, usize_t blksz) -{ - usize_t mod = sz & (blksz-1); - - XD3_ASSERT (xd3_check_pow2 (blksz, NULL) == 0); - - return mod ? (sz + (blksz - mod)) : sz; -} - -/*********************************************************************** - Adler32 stream function: code copied from Zlib, defined in RFC1950 - ***********************************************************************/ - -#define A32_BASE 65521L /* Largest prime smaller than 2^16 */ -#define A32_NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ - -#define A32_DO1(buf,i) {s1 += buf[i]; s2 += s1;} -#define A32_DO2(buf,i) A32_DO1(buf,i); A32_DO1(buf,i+1); -#define A32_DO4(buf,i) A32_DO2(buf,i); A32_DO2(buf,i+2); -#define A32_DO8(buf,i) A32_DO4(buf,i); A32_DO4(buf,i+4); -#define A32_DO16(buf) A32_DO8(buf,0); A32_DO8(buf,8); - -static unsigned long adler32 (unsigned long adler, const uint8_t *buf, usize_t len) -{ - unsigned long s1 = adler & 0xffff; - unsigned long s2 = (adler >> 16) & 0xffff; - int k; - - while (len > 0) - { - k = (len < A32_NMAX) ? len : A32_NMAX; - len -= k; - - while (k >= 16) - { - A32_DO16(buf); - buf += 16; - k -= 16; - } - - if (k != 0) - { - do - { - s1 += *buf++; - s2 += s1; - } - while (--k); - } - - s1 %= A32_BASE; - s2 %= A32_BASE; - } - - return (s2 << 16) | s1; -} - -/*********************************************************************** - Run-length function - ***********************************************************************/ - -#if XD3_ENCODER -static int -xd3_comprun (const uint8_t *seg, int slook, uint8_t *run_cp) -{ - int i; - int run_l = 0; - uint8_t run_c = 0; - - for (i = 0; i < slook; i += 1) - { - NEXTRUN(seg[i]); - } - - (*run_cp) = run_c; - - return run_l; -} -#endif - -/*********************************************************************** - Basic encoder/decoder functions - ***********************************************************************/ - -static inline int -xd3_decode_byte (xd3_stream *stream, usize_t *val) -{ - if (stream->avail_in == 0) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - (*val) = stream->next_in[0]; - - DECODE_INPUT (1); - return 0; -} - -static inline int -xd3_decode_bytes (xd3_stream *stream, uint8_t *buf, usize_t *pos, usize_t size) -{ - usize_t want; - usize_t take; - - /* Note: The case where (*pos == size) happens when a zero-length appheader or code - * table is transmitted, but there is nothing in the standard against that. */ - - while (*pos < size) - { - if (stream->avail_in == 0) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - want = size - *pos; - take = min (want, stream->avail_in); - - memcpy (buf + *pos, stream->next_in, take); - - DECODE_INPUT (take); - (*pos) += take; - } - - return 0; -} - -#if XD3_ENCODER -static inline int -xd3_emit_byte (xd3_stream *stream, - xd3_output **outputp, - uint8_t code) -{ - xd3_output *output = (*outputp); - - if (output->next == output->avail) - { - xd3_output *aoutput; - - if ((aoutput = xd3_alloc_output (stream, output)) == NULL) - { - return ENOMEM; - } - - output = (*outputp) = aoutput; - } - - output->base[output->next++] = code; - - return 0; -} - -static inline int -xd3_emit_bytes (xd3_stream *stream, - xd3_output **outputp, - const uint8_t *base, - usize_t size) -{ - xd3_output *output = (*outputp); - - do - { - usize_t take; - - if (output->next == output->avail) - { - xd3_output *aoutput; - - if ((aoutput = xd3_alloc_output (stream, output)) == NULL) - { - return ENOMEM; - } - - output = (*outputp) = aoutput; - } - - take = min (output->avail - output->next, size); - - memcpy (output->base + output->next, base, take); - - output->next += take; - size -= take; - base += take; - } - while (size > 0); - - return 0; -} -#endif /* XD3_ENCODER */ - -/********************************************************************* - Integer encoder/decoder functions - **********************************************************************/ - -#define DECODE_INTEGER_TYPE(PART,OFLOW) \ - while (stream->avail_in != 0) \ - { \ - usize_t next = stream->next_in[0]; \ - \ - DECODE_INPUT(1); \ - \ - if (PART & OFLOW) \ - { \ - stream->msg = "overflow in decode_integer"; \ - return XD3_INVALID_INPUT; \ - } \ - \ - PART = (PART << 7) | (next & 127); \ - \ - if ((next & 128) == 0) \ - { \ - (*val) = PART; \ - PART = 0; \ - return 0; \ - } \ - } \ - \ - stream->msg = "further input required"; \ - return XD3_INPUT - -#define READ_INTEGER_TYPE(TYPE, OFLOW) \ - TYPE val = 0; \ - const uint8_t *inp = (*inpp); \ - usize_t next; \ - \ - do \ - { \ - if (inp == max) \ - { \ - stream->msg = "end-of-input in read_integer"; \ - return XD3_INVALID_INPUT; \ - } \ - \ - if (val & OFLOW) \ - { \ - stream->msg = "overflow in read_intger"; \ - return XD3_INVALID_INPUT; \ - } \ - \ - next = (*inp++); \ - val = (val << 7) | (next & 127); \ - } \ - while (next & 128); \ - \ - (*valp) = val; \ - (*inpp) = inp; \ - \ - return 0 - -#define EMIT_INTEGER_TYPE() \ - /* max 64-bit value in base-7 encoding is 9.1 bytes */ \ - uint8_t buf[10]; \ - usize_t bufi = 10; \ - \ - XD3_ASSERT (num >= 0); \ - \ - /* This loop performs division and turns on all MSBs. */ \ - do \ - { \ - buf[--bufi] = (num & 127) | 128; \ - num >>= 7; \ - } \ - while (num != 0); \ - \ - /* Turn off MSB of the last byte. */ \ - buf[9] &= 127; \ - \ - XD3_ASSERT (bufi >= 0); \ - \ - return xd3_emit_bytes (stream, output, buf + bufi, 10 - bufi) - -#define IF_SIZEOF32(x) if (num < (1U << (7 * (x)))) return (x); -#define IF_SIZEOF64(x) if (num < (1ULL << (7 * (x)))) return (x); - -#if USE_UINT32 -static inline uint32_t -xd3_sizeof_uint32_t (uint32_t num) -{ - IF_SIZEOF32(1); - IF_SIZEOF32(2); - IF_SIZEOF32(3); - IF_SIZEOF32(4); - return 5; -} - -static inline int -xd3_decode_uint32_t (xd3_stream *stream, uint32_t *val) -{ DECODE_INTEGER_TYPE (stream->dec_32part, UINT32_OFLOW_MASK); } - -static inline int -xd3_read_uint32_t (xd3_stream *stream, const uint8_t **inpp, - const uint8_t *max, uint32_t *valp) -{ READ_INTEGER_TYPE (uint32_t, UINT32_OFLOW_MASK); } - -#if XD3_ENCODER -static inline int -xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, uint32_t num) -{ EMIT_INTEGER_TYPE (); } -#endif -#endif - -#if USE_UINT64 -static inline int -xd3_decode_uint64_t (xd3_stream *stream, uint64_t *val) -{ DECODE_INTEGER_TYPE (stream->dec_64part, UINT64_OFLOW_MASK); } - -#if XD3_ENCODER -static inline int -xd3_emit_uint64_t (xd3_stream *stream, xd3_output **output, uint64_t num) -{ EMIT_INTEGER_TYPE (); } -#endif - -/* These are tested but not used */ -#if REGRESSION_TEST -static int -xd3_read_uint64_t (xd3_stream *stream, const uint8_t **inpp, - const uint8_t *max, uint64_t *valp) -{ READ_INTEGER_TYPE (uint64_t, UINT64_OFLOW_MASK); } - -static uint32_t -xd3_sizeof_uint64_t (uint64_t num) -{ - IF_SIZEOF64(1); - IF_SIZEOF64(2); - IF_SIZEOF64(3); - IF_SIZEOF64(4); - IF_SIZEOF64(5); - IF_SIZEOF64(6); - IF_SIZEOF64(7); - IF_SIZEOF64(8); - IF_SIZEOF64(9); - - return 10; -} -#endif - -#endif - -/*********************************************************************** - Address cache stuff - ***********************************************************************/ - -static int -xd3_alloc_cache (xd3_stream *stream) -{ - if (stream->acache.near_array != NULL) - { - xd3_free (stream, stream->acache.near_array); - } - - if (stream->acache.same_array != NULL) - { - xd3_free (stream, stream->acache.same_array); - } - - if (((stream->acache.s_near > 0) && - (stream->acache.near_array = (usize_t*) - xd3_alloc (stream, stream->acache.s_near, sizeof (usize_t))) - == NULL) || - ((stream->acache.s_same > 0) && - (stream->acache.same_array = (usize_t*) - xd3_alloc (stream, stream->acache.s_same * 256, sizeof (usize_t))) - == NULL)) - { - return ENOMEM; - } - - return 0; -} - -void -xd3_init_cache (xd3_addr_cache* acache) -{ - if (acache->s_near > 0) - { - memset (acache->near_array, 0, acache->s_near * sizeof (usize_t)); - acache->next_slot = 0; - } - - if (acache->s_same > 0) - { - memset (acache->same_array, 0, acache->s_same * 256 * sizeof (usize_t)); - } -} - -static void -xd3_update_cache (xd3_addr_cache* acache, usize_t addr) -{ - if (acache->s_near > 0) - { - acache->near_array[acache->next_slot] = addr; - acache->next_slot = (acache->next_slot + 1) % acache->s_near; - } - - if (acache->s_same > 0) - { - acache->same_array[addr % (acache->s_same*256)] = addr; - } -} - -#if XD3_ENCODER -/* OPT: this gets called a lot, can it be optimized? */ -static int -xd3_encode_address (xd3_stream *stream, usize_t addr, usize_t here, uint8_t* mode) -{ - usize_t d, bestd; - usize_t i, bestm, ret; - xd3_addr_cache* acache = & stream->acache; - -#define SMALLEST_INT(x) do { if (((x) & ~127) == 0) { goto good; } } while (0) - - /* Attempt to find the address mode that yields the smallest integer value - * for "d", the encoded address value, thereby minimizing the encoded size - * of the address. */ - bestd = addr; - bestm = VCD_SELF; - - XD3_ASSERT (addr < here); - - SMALLEST_INT (bestd); - - if ((d = here-addr) < bestd) - { - bestd = d; - bestm = VCD_HERE; - - SMALLEST_INT (bestd); - } - - for (i = 0; i < acache->s_near; i += 1) - { - d = addr - acache->near_array[i]; - - if (d >= 0 && d < bestd) - { - bestd = d; - bestm = i+2; /* 2 counts the VCD_SELF, VCD_HERE modes */ - - SMALLEST_INT (bestd); - } - } - - if (acache->s_same > 0 && acache->same_array[d = addr%(acache->s_same*256)] == addr) - { - bestd = d%256; - bestm = acache->s_near + 2 + d/256; /* 2 + s_near offsets past the VCD_NEAR modes */ - - if ((ret = xd3_emit_byte (stream, & ADDR_TAIL (stream), bestd))) { return ret; } - } - else - { - good: - - if ((ret = xd3_emit_size (stream, & ADDR_TAIL (stream), bestd))) { return ret; } - } - - xd3_update_cache (acache, addr); - - (*mode) += bestm; - - return 0; -} -#endif - -static int -xd3_decode_address (xd3_stream *stream, usize_t here, - usize_t mode, const uint8_t **inpp, - const uint8_t *max, uint32_t *valp) -{ - int ret; - usize_t same_start = 2 + stream->acache.s_near; - - if (mode < same_start) - { - if ((ret = xd3_read_size (stream, inpp, max, valp))) { return ret; } - - switch (mode) - { - case VCD_SELF: - break; - case VCD_HERE: - (*valp) = here - (*valp); - break; - default: - (*valp) += stream->acache.near_array[mode - 2]; - break; - } - } - else - { - if (*inpp == max) - { - stream->msg = "address underflow"; - return XD3_INVALID_INPUT; - } - - mode -= same_start; - - (*valp) = stream->acache.same_array[mode*256 + (**inpp)]; - - (*inpp) += 1; - } - - xd3_update_cache (& stream->acache, *valp); - - return 0; -} - -/*********************************************************************** - Alloc/free -***********************************************************************/ - -static void* -__xd3_alloc_func (void* opaque, usize_t items, usize_t size) -{ - return malloc (items * size); -} - -static void -__xd3_free_func (void* opaque, void* address) -{ - free (address); -} - -static void* -xd3_alloc (xd3_stream *stream, - usize_t elts, - usize_t size) -{ - void *a = stream->alloc (stream->opaque, elts, size); - - if (a != NULL) - { - IF_DEBUG (stream->alloc_cnt += 1); - IF_DEBUG2 (DP(RINT "[stream %p malloc] size %u ptr %p\n", - stream, elts * size, a)); - } - else - { - stream->msg = "out of memory"; - } - - return a; -} - -static void -xd3_free (xd3_stream *stream, - void *ptr) -{ - if (ptr != NULL) - { - IF_DEBUG (stream->free_cnt += 1); - XD3_ASSERT (stream->free_cnt <= stream->alloc_cnt); - IF_DEBUG2 (DP(RINT "[stream %p free] %p\n", - stream, ptr)); - stream->free (stream->opaque, ptr); - } -} - -#if XD3_ENCODER -static void* -xd3_alloc0 (xd3_stream *stream, - usize_t elts, - usize_t size) -{ - void *a = xd3_alloc (stream, elts, size); - - if (a != NULL) - { - memset (a, 0, elts * size); - } - - return a; -} - -static xd3_output* -xd3_alloc_output (xd3_stream *stream, - xd3_output *old_output) -{ - xd3_output *output; - uint8_t *base; - - if (stream->enc_free != NULL) - { - output = stream->enc_free; - stream->enc_free = output->next_page; - } - else - { - if ((output = (xd3_output*) xd3_alloc (stream, 1, sizeof (xd3_output))) == NULL) - { - return NULL; - } - - if ((base = (uint8_t*) xd3_alloc (stream, XD3_ALLOCSIZE, sizeof (uint8_t))) == NULL) - { - xd3_free (stream, output); - return NULL; - } - - output->base = base; - output->avail = XD3_ALLOCSIZE; - } - - output->next = 0; - - if (old_output) - { - old_output->next_page = output; - } - - output->next_page = NULL; - - return output; -} - -static usize_t -xd3_sizeof_output (xd3_output *output) -{ - usize_t s = 0; - - for (; output; output = output->next_page) - { - s += output->next; - } - - return s; -} - -static void -xd3_freelist_output (xd3_stream *stream, - xd3_output *output) -{ - xd3_output *tmp; - - while (output) - { - tmp = output; - output = output->next_page; - - tmp->next = 0; - tmp->next_page = stream->enc_free; - stream->enc_free = tmp; - } -} - -static void -xd3_free_output (xd3_stream *stream, - xd3_output *output) -{ - xd3_output *next; - - again: - if (output == NULL) - { - return; - } - - next = output->next_page; - - xd3_free (stream, output->base); - xd3_free (stream, output); - - output = next; - goto again; -} -#endif /* XD3_ENCODER */ - -void -xd3_free_stream (xd3_stream *stream) -{ - xd3_iopt_buflist *blist = stream->iopt_alloc; - - while (blist != NULL) - { - xd3_iopt_buflist *tmp = blist; - blist = blist->next; - xd3_free (stream, tmp->buffer); - xd3_free (stream, tmp); - } - - xd3_free (stream, stream->large_table); - xd3_free (stream, stream->small_table); - xd3_free (stream, stream->small_prev); - -#if XD3_ENCODER - { - int i; - for (i = 0; i < ENC_SECTS; i += 1) - { - xd3_free_output (stream, stream->enc_heads[i]); - } - xd3_free_output (stream, stream->enc_free); - } -#endif - - xd3_free (stream, stream->acache.near_array); - xd3_free (stream, stream->acache.same_array); - - xd3_free (stream, stream->inst_sect.copied1); - xd3_free (stream, stream->addr_sect.copied1); - xd3_free (stream, stream->data_sect.copied1); - - xd3_free (stream, stream->dec_buffer); - xd3_free (stream, (uint8_t*) stream->dec_lastwin); - - xd3_free (stream, stream->buf_in); - xd3_free (stream, stream->dec_appheader); - xd3_free (stream, stream->dec_codetbl); - xd3_free (stream, stream->code_table_alloc); - -#if SECONDARY_ANY - xd3_free (stream, stream->inst_sect.copied2); - xd3_free (stream, stream->addr_sect.copied2); - xd3_free (stream, stream->data_sect.copied2); - - if (stream->sec_type != NULL) - { - stream->sec_type->destroy (stream, stream->sec_stream_d); - stream->sec_type->destroy (stream, stream->sec_stream_i); - stream->sec_type->destroy (stream, stream->sec_stream_a); - } -#endif - - xd3_free (stream, stream->whole_target.adds); - xd3_free (stream, stream->whole_target.inst); - xd3_free (stream, stream->whole_target.wininfo); - - XD3_ASSERT (stream->alloc_cnt == stream->free_cnt); - - memset (stream, 0, sizeof (xd3_stream)); -} - -#if (XD3_DEBUG > 1 || VCDIFF_TOOLS) -static const char* -xd3_rtype_to_string (xd3_rtype type, int print_mode) -{ - switch (type) - { - case XD3_NOOP: - return "NOOP "; - case XD3_RUN: - return "RUN "; - case XD3_ADD: - return "ADD "; - default: break; - } - if (! print_mode) - { - return "CPY "; - } - switch (type) - { - case XD3_CPY + 0: return "CPY_0"; - case XD3_CPY + 1: return "CPY_1"; - case XD3_CPY + 2: return "CPY_2"; - case XD3_CPY + 3: return "CPY_3"; - case XD3_CPY + 4: return "CPY_4"; - case XD3_CPY + 5: return "CPY_5"; - case XD3_CPY + 6: return "CPY_6"; - case XD3_CPY + 7: return "CPY_7"; - case XD3_CPY + 8: return "CPY_8"; - case XD3_CPY + 9: return "CPY_9"; - default: return "CPY>9"; - } -} -#endif - -/**************************************************************** - Stream configuration - ******************************************************************/ - -int -xd3_config_stream(xd3_stream *stream, - xd3_config *config) -{ - int ret; - xd3_config defcfg; - xd3_smatcher *smatcher = &stream->smatcher; - - if (config == NULL) - { - config = & defcfg; - memset (config, 0, sizeof (*config)); - } - - /* Initial setup: no error checks yet */ - memset (stream, 0, sizeof (*stream)); - - stream->winsize = config->winsize ? config->winsize : XD3_DEFAULT_WINSIZE; - stream->sprevsz = config->sprevsz ? config->sprevsz : XD3_DEFAULT_SPREVSZ; - stream->srcwin_maxsz = config->srcwin_maxsz ? - config->srcwin_maxsz : XD3_DEFAULT_SRCWINSZ; - - if (config->iopt_size == 0) - { - stream->iopt_size = XD3_ALLOCSIZE / sizeof(xd3_rinst); - stream->iopt_unlimited = 1; - } - else - { - stream->iopt_size = config->iopt_size; - } - - stream->getblk = config->getblk; - stream->alloc = config->alloc ? config->alloc : __xd3_alloc_func; - stream->free = config->freef ? config->freef : __xd3_free_func; - stream->opaque = config->opaque; - stream->flags = config->flags; - - /* Secondary setup. */ - stream->sec_data = config->sec_data; - stream->sec_inst = config->sec_inst; - stream->sec_addr = config->sec_addr; - - stream->sec_data.data_type = DATA_SECTION; - stream->sec_inst.data_type = INST_SECTION; - stream->sec_addr.data_type = ADDR_SECTION; - - /* Check static sizes. */ - if (sizeof (usize_t) != SIZEOF_USIZE_T || - sizeof (xoff_t) != SIZEOF_XOFF_T || - (ret = xd3_check_pow2(XD3_ALLOCSIZE, NULL))) - { - stream->msg = "incorrect compilation: wrong integer sizes"; - return XD3_INTERNAL; - } - - /* Check/set secondary compressor. */ - switch (stream->flags & XD3_SEC_TYPE) - { - case 0: - if (stream->flags & XD3_SEC_NOALL) - { - stream->msg = "XD3_SEC flags require a secondary compressor type"; - return XD3_INTERNAL; - } - break; - case XD3_SEC_FGK: - FGK_CASE (stream); - case XD3_SEC_DJW: - DJW_CASE (stream); - default: - stream->msg = "too many secondary compressor types set"; - return XD3_INTERNAL; - } - - /* Check/set encoder code table. */ - switch (stream->flags & XD3_ALT_CODE_TABLE) { - case 0: - stream->code_table_desc = & __rfc3284_code_table_desc; - stream->code_table_func = xd3_rfc3284_code_table; - break; -#if GENERIC_ENCODE_TABLES - case XD3_ALT_CODE_TABLE: - stream->code_table_desc = & __alternate_code_table_desc; - stream->code_table_func = xd3_alternate_code_table; - stream->comp_table_func = xd3_compute_alternate_table_encoding; - break; -#endif - default: - stream->msg = "alternate code table support was not compiled"; - return XD3_INTERNAL; - } - - /* Check sprevsz */ - if (smatcher->small_chain == 1 && - smatcher->small_lchain == 1) - { - stream->sprevsz = 0; - } - else - { - if ((ret = xd3_check_pow2 (stream->sprevsz, NULL))) - { - stream->msg = "sprevsz is required to be a power of two"; - return XD3_INTERNAL; - } - - stream->sprevmask = stream->sprevsz - 1; - } - - /* Default scanner settings. */ -#if XD3_ENCODER - switch (config->smatch_cfg) - { - IF_BUILD_SOFT(case XD3_SMATCH_SOFT: - { - *smatcher = config->smatcher_soft; - smatcher->string_match = __smatcher_soft.string_match; - smatcher->name = __smatcher_soft.name; - if (smatcher->large_look < MIN_MATCH || - smatcher->large_step < 1 || - smatcher->small_look < MIN_MATCH) - { - stream->msg = "invalid soft string-match config"; - return XD3_INVALID; - } - break; - }) - - IF_BUILD_DEFAULT(case XD3_SMATCH_DEFAULT: - *smatcher = __smatcher_default; - break;) - IF_BUILD_SLOW(case XD3_SMATCH_SLOW: - *smatcher = __smatcher_slow; - break;) - IF_BUILD_FASTEST(case XD3_SMATCH_FASTEST: - *smatcher = __smatcher_fastest; - break;) - IF_BUILD_FASTER(case XD3_SMATCH_FASTER: - *smatcher = __smatcher_faster; - break;) - IF_BUILD_FAST(case XD3_SMATCH_FAST: - *smatcher = __smatcher_fast; - break;) - default: - stream->msg = "invalid string match config type"; - return XD3_INTERNAL; - } - - if (config->smatch_cfg == XD3_SMATCH_DEFAULT && - (stream->flags & XD3_COMPLEVEL_MASK) != 0) - { - int level = (stream->flags & XD3_COMPLEVEL_MASK) >> XD3_COMPLEVEL_SHIFT; - - switch (level) - { - case 1: - IF_BUILD_FASTEST(*smatcher = __smatcher_fastest; - break;) - case 2: - IF_BUILD_FASTER(*smatcher = __smatcher_faster; - break;) - case 3: case 4: case 5: - IF_BUILD_FAST(*smatcher = __smatcher_fast; - break;) - case 6: - IF_BUILD_DEFAULT(*smatcher = __smatcher_default; - break;) - default: - IF_BUILD_SLOW(*smatcher = __smatcher_slow; - break;) - IF_BUILD_DEFAULT(*smatcher = __smatcher_default; - break;) - IF_BUILD_FAST(*smatcher = __smatcher_fast; - break;) - IF_BUILD_FASTER(*smatcher = __smatcher_faster; - break;) - IF_BUILD_FASTEST(*smatcher = __smatcher_fastest; - break;) - } - } -#endif - - return 0; -} - -/***************************************************************** - Getblk interface - ***********************************************************/ - -/* This function interfaces with the client getblk function, checks - * its results, etc. */ -static int -xd3_getblk (xd3_stream *stream, xoff_t blkno) -{ - int ret; - xd3_source *source = stream->src; - - if (source->curblk == NULL || - blkno != source->curblkno) - { - if (blkno >= source->blocks) - { - stream->msg = "source file too short"; - return XD3_INTERNAL; - } - - XD3_ASSERT (source->curblk != NULL || blkno != source->curblkno); - - source->getblkno = blkno; - - if (stream->getblk == NULL) - { - stream->msg = "getblk source input"; - return XD3_GETSRCBLK; - } - else if ((ret = stream->getblk (stream, source, blkno)) != 0) - { - stream->msg = "getblk failed"; - return ret; - } - - XD3_ASSERT (source->curblk != NULL); - } - - if (source->onblk != (blkno == source->blocks - 1 ? - source->onlastblk : source->blksize)) - { - stream->msg = "getblk returned short block"; - return XD3_INTERNAL; - } - - return 0; -} - -/*********************************************************** - Stream open/close - ***************************************************************/ - -int -xd3_set_source (xd3_stream *stream, - xd3_source *src) -{ - xoff_t blk_num; - usize_t tail_size, shiftby; - - IF_DEBUG1 (DP(RINT "[set source] size %"Q"u\n", src->size)); - - if (src == NULL || src->size < stream->smatcher.large_look) { return 0; } - - stream->src = src; - - // If src->blksize is a power-of-two, xd3_blksize_div() will use - // shift and mask rather than divide. Check that here. - if (xd3_check_pow2 (src->blksize, &shiftby) == 0) - { - src->shiftby = shiftby; - src->maskby = (1 << shiftby) - 1; - } - else if (src->size <= src->blksize) - { - int x = xd3_pow2_roundup (src->blksize); - xd3_check_pow2 (x, &shiftby); - src->shiftby = shiftby; - src->maskby = (1 << shiftby) - 1; - } - else - { - src->shiftby = 0; - src->maskby = 0; - } - - xd3_blksize_div (src->size, src, &blk_num, &tail_size); - src->blocks = blk_num + (tail_size > 0); - src->onlastblk = xd3_bytes_on_srcblk (src, src->blocks - 1); - src->srclen = 0; - src->srcbase = 0; - - return 0; -} - -void -xd3_abort_stream (xd3_stream *stream) -{ - stream->dec_state = DEC_ABORTED; - stream->enc_state = ENC_ABORTED; -} - -int -xd3_close_stream (xd3_stream *stream) -{ - if (stream->enc_state != 0 && stream->enc_state != ENC_ABORTED) - { - if (stream->buf_leftover != NULL) - { - stream->msg = "encoding is incomplete"; - return XD3_INTERNAL; - } - - if (stream->enc_state == ENC_POSTWIN) - { -#if XD3_ENCODER - xd3_encode_reset (stream); -#endif - stream->current_window += 1; - stream->enc_state = ENC_INPUT; - } - - /* If encoding, should be ready for more input but not actually - have any. */ - if (stream->enc_state != ENC_INPUT || stream->avail_in != 0) - { - stream->msg = "encoding is incomplete"; - return XD3_INTERNAL; - } - } - else - { - switch (stream->dec_state) - { - case DEC_VCHEAD: - case DEC_WININD: - /* TODO: Address the zero-byte ambiguity. Does the encoder - * emit a window or not? If so, then catch an error here. - * If not, need another routine to say - * decode_at_least_one_if_empty. */ - case DEC_ABORTED: - break; - default: - /* If decoding, should be ready for the next window. */ - stream->msg = "EOF in decode"; - return XD3_INTERNAL; - } - } - - return 0; -} - -/************************************************************** - Application header - ****************************************************************/ - -int -xd3_get_appheader (xd3_stream *stream, - uint8_t **data, - usize_t *size) -{ - if (stream->dec_state < DEC_WININD) - { - stream->msg = "application header not available"; - return XD3_INTERNAL; - } - - (*data) = stream->dec_appheader; - (*size) = stream->dec_appheadsz; - return 0; -} - -/********************************************************** - Decoder stuff - *************************************************/ - -#include "xdelta3-decode.h" - -/**************************************************************** - Encoder stuff - *****************************************************************/ - -#if XD3_ENCODER -void -xd3_set_appheader (xd3_stream *stream, - const uint8_t *data, - usize_t size) -{ - stream->enc_appheader = data; - stream->enc_appheadsz = size; -} - -#if XD3_DEBUG -static int -xd3_iopt_check (xd3_stream *stream) -{ - usize_t ul = xd3_rlist_length (& stream->iopt_used); - usize_t fl = xd3_rlist_length (& stream->iopt_free); - - return (ul + fl + (stream->iout ? 1 : 0)) == stream->iopt_size; -} -#endif - -static xd3_rinst* -xd3_iopt_free (xd3_stream *stream, xd3_rinst *i) -{ - xd3_rinst *n = xd3_rlist_remove (i); - xd3_rlist_push_back (& stream->iopt_free, i); - return n; -} - -static void -xd3_iopt_free_nonadd (xd3_stream *stream, xd3_rinst *i) -{ - if (i->type != XD3_ADD) - { - xd3_rlist_push_back (& stream->iopt_free, i); - } -} - -/* When an instruction is ready to flush from the iopt buffer, this - * function is called to produce an encoding. It writes the - * instruction plus size, address, and data to the various encoding - * sections. */ -static int -xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) -{ - int ret; - - /* Check for input overflow. */ - XD3_ASSERT (inst->pos + inst->size <= stream->avail_in); - - switch (inst->type) - { - case XD3_CPY: - { - /* the address may have an offset if there is a source window. */ - usize_t addr; - xd3_source *src = stream->src; - - if (src != NULL) - { - /* If there is a source copy, the source must have its - * source window decided before we can encode. This can - * be bad -- we have to make this decision even if no - * source matches have been found. */ - if (stream->srcwin_decided == 0) - { - if ((ret = xd3_srcwin_setup (stream))) { return ret; } - } - - /* xtra field indicates the copy is from the source */ - if (inst->xtra) - { - XD3_ASSERT (inst->addr >= src->srcbase); - XD3_ASSERT (inst->addr + inst->size <= src->srcbase + src->srclen); - addr = (inst->addr - src->srcbase); - stream->n_scpy += 1; - stream->l_scpy += inst->size; - } - else - { - /* with source window: target copy address is offset by taroff. */ - addr = stream->taroff + (usize_t) inst->addr; - stream->n_tcpy += 1; - stream->l_tcpy += inst->size; - } - } - else - { - addr = (usize_t) inst->addr; - stream->n_tcpy += 1; - stream->l_tcpy += inst->size; - } - - /* Note: used to assert inst->size >= MIN_MATCH, but not true - * for merge operations & identical match heuristics. */ - /* the "here" position is always offset by taroff */ - if ((ret = xd3_encode_address (stream, addr, inst->pos + stream->taroff, - & inst->type))) - { - return ret; - } - - IF_DEBUG1 ({ - static int cnt; - DP(RINT "[iopt copy:%d] pos %"Q"u-%"Q"u addr %"Q"u-%"Q"u size %u\n", - cnt++, - stream->total_in + inst->pos, - stream->total_in + inst->pos + inst->size, - inst->addr, inst->addr + inst->size, inst->size); - }); - break; - } - case XD3_RUN: - { - XD3_ASSERT (inst->size >= MIN_MATCH); - - if ((ret = xd3_emit_byte (stream, & DATA_TAIL (stream), inst->xtra))) { return ret; } - - stream->n_run += 1; - stream->l_run += inst->size; - - IF_DEBUG1 ({ - static int cnt; - DP(RINT "[iopt run:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); - }); - break; - } - case XD3_ADD: - { - if ((ret = xd3_emit_bytes (stream, & DATA_TAIL (stream), - stream->next_in + inst->pos, inst->size))) { return ret; } - - stream->n_add += 1; - stream->l_add += inst->size; - - IF_DEBUG1 ({ - static int cnt; - DP(RINT "[iopt add:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); - }); - - break; - } - } - - /* This is the only place stream->unencoded_offset is incremented. */ - XD3_ASSERT (stream->unencoded_offset == inst->pos); - stream->unencoded_offset += inst->size; - - inst->code2 = 0; - - XD3_CHOOSE_INSTRUCTION (stream, stream->iout, inst); - - if (stream->iout != NULL) - { - if (stream->iout->code2 != 0) - { - if ((ret = xd3_emit_double (stream, stream->iout, inst, stream->iout->code2))) { return ret; } - - xd3_iopt_free_nonadd (stream, stream->iout); - xd3_iopt_free_nonadd (stream, inst); - stream->iout = NULL; - return 0; - } - else - { - if ((ret = xd3_emit_single (stream, stream->iout, stream->iout->code1))) { return ret; } - - xd3_iopt_free_nonadd (stream, stream->iout); - } - } - - stream->iout = inst; - - return 0; -} - -/* This possibly encodes an add instruction, iadd, which must remain - * on the stack until the following call to - * xd3_iopt_finish_encoding. */ -static int -xd3_iopt_add (xd3_stream *stream, usize_t pos, xd3_rinst *iadd) -{ - int ret; - usize_t off = stream->unencoded_offset; - - if (pos > off) - { - iadd->type = XD3_ADD; - iadd->pos = off; - iadd->size = pos - off; - - if ((ret = xd3_iopt_finish_encoding (stream, iadd))) { return ret; } - } - - return 0; -} - -/* This function calls xd3_iopt_finish_encoding to finish encoding an - * instruction, and it may also produce an add instruction for an - * unmatched region. */ -static int -xd3_iopt_add_encoding (xd3_stream *stream, xd3_rinst *inst) -{ - int ret; - xd3_rinst iadd; - - if ((ret = xd3_iopt_add (stream, inst->pos, & iadd))) { return ret; } - - if ((ret = xd3_iopt_finish_encoding (stream, inst))) { return ret; } - - return 0; -} - -/* Generates a final add instruction to encode the remaining input. */ -static int -xd3_iopt_add_finalize (xd3_stream *stream) -{ - int ret; - xd3_rinst iadd; - - if ((ret = xd3_iopt_add (stream, stream->avail_in, & iadd))) { return ret; } - - if (stream->iout) - { - if ((ret = xd3_emit_single (stream, stream->iout, stream->iout->code1))) { return ret; } - - xd3_iopt_free_nonadd (stream, stream->iout); - stream->iout = NULL; - } - - return 0; -} - -/* Compact the instruction buffer by choosing the best non-overlapping - * instructions when lazy string-matching. There are no ADDs in the - * iopt buffer because those are synthesized in xd3_iopt_add_encoding - * and during xd3_iopt_add_finalize. */ -static int -xd3_iopt_flush_instructions (xd3_stream *stream, int force) -{ - xd3_rinst *r1 = xd3_rlist_front (& stream->iopt_used); - xd3_rinst *r2; - xd3_rinst *r3; - usize_t r1end; - usize_t r2end; - usize_t r2off; - usize_t r2moff; - usize_t gap; - usize_t flushed; - int ret; - - XD3_ASSERT (xd3_iopt_check (stream)); - - /* Note: once tried to skip this step if it's possible to assert - * there are no overlapping instructions. Doesn't work because - * xd3_opt_erase leaves overlapping instructions. */ - while (! xd3_rlist_end (& stream->iopt_used, r1) && - ! xd3_rlist_end (& stream->iopt_used, r2 = xd3_rlist_next (r1))) - { - r1end = r1->pos + r1->size; - - /* If the instructions do not overlap, continue. */ - if (r1end <= r2->pos) - { - r1 = r2; - continue; - } - - r2end = r2->pos + r2->size; - - /* The min_match adjustments prevent this. */ - XD3_ASSERT (r2end > (r1end + LEAST_MATCH_INCR)); - - /* If r3 is available... */ - if (! xd3_rlist_end (& stream->iopt_used, r3 = xd3_rlist_next (r2))) - { - /* If r3 starts before r1 finishes or just about, r2 is irrelevant */ - if (r3->pos <= r1end + 1) - { - xd3_iopt_free (stream, r2); - continue; - } - } - else if (! force) - { - /* Unless force, end the loop when r3 is not available. */ - break; - } - - r2off = r2->pos - r1->pos; - r2moff = r2end - r1end; - gap = r2end - r1->pos; - - /* If the two matches overlap almost entirely, choose the better match - * and discard the other. The else branch can still create inefficient - * copies, e.g., a 4-byte copy that takes 4 bytes to encode, which - * xd3_smatch() wouldn't allow by its crude efficiency check. However, - * in this case there are adjacent copies which mean the add would cost - * one extra byte. Allow the inefficiency here. */ - if (gap < 2*MIN_MATCH || r2moff <= 2 || r2off <= 2) - { - /* Only one match should be used, choose the longer one. */ - if (r1->size < r2->size) - { - xd3_iopt_free (stream, r1); - r1 = r2; - } - else - { - /* We are guaranteed that r1 does not overlap now, so advance past r2 */ - r1 = xd3_iopt_free (stream, r2); - } - continue; - } - else - { - /* Shorten one of the instructions -- could be optimized - * based on the address cache. */ - usize_t average; - usize_t newsize; - usize_t adjust1; - - XD3_ASSERT (r1end > r2->pos && r2end > r1->pos); - - /* Try to balance the length of both instructions, but avoid - * making both longer than MAX_MATCH_SPLIT . */ - average = gap / 2; - newsize = min (MAX_MATCH_SPLIT, gap - average); - - /* Should be possible to simplify this code. */ - if (newsize > r1->size) - { - /* shorten r2 */ - adjust1 = r1end - r2->pos; - } - else if (newsize > r2->size) - { - /* shorten r1 */ - adjust1 = r1end - r2->pos; - - XD3_ASSERT (r1->size > adjust1); - - r1->size -= adjust1; - - /* don't shorten r2 */ - adjust1 = 0; - } - else - { - /* shorten r1 */ - adjust1 = r1->size - newsize; - - if (r2->pos > r1end - adjust1) - { - adjust1 -= r2->pos - (r1end - adjust1); - } - - XD3_ASSERT (r1->size > adjust1); - - r1->size -= adjust1; - - /* shorten r2 */ - XD3_ASSERT (r1->pos + r1->size >= r2->pos); - - adjust1 = r1->pos + r1->size - r2->pos; - } - - /* Fallthrough above if-else, shorten r2 */ - XD3_ASSERT (r2->size > adjust1); - - r2->size -= adjust1; - r2->pos += adjust1; - r2->addr += adjust1; - - XD3_ASSERT (r1->size >= MIN_MATCH); - XD3_ASSERT (r2->size >= MIN_MATCH); - - r1 = r2; - } - } - - XD3_ASSERT (xd3_iopt_check (stream)); - - /* If forcing, pick instructions until the list is empty, otherwise - * this empties 50% of the queue. */ - for (flushed = 0; ! xd3_rlist_empty (& stream->iopt_used); ) - { - xd3_rinst *renc = xd3_rlist_pop_front (& stream->iopt_used); - if ((ret = xd3_iopt_add_encoding (stream, renc))) - { - return ret; - } - - if (! force) - { - if (++flushed > stream->iopt_size / 2) - { - break; - } - - /* If there are only two instructions remaining, break, - * because they were not optimized. This means there were - * more than 50% eliminated by the loop above. */ - r1 = xd3_rlist_front (& stream->iopt_used); - if (xd3_rlist_end(& stream->iopt_used, r1) || - xd3_rlist_end(& stream->iopt_used, r2 = xd3_rlist_next (r1)) || - xd3_rlist_end(& stream->iopt_used, r3 = xd3_rlist_next (r2))) - { - break; - } - } - } - - XD3_ASSERT (xd3_iopt_check (stream)); - - XD3_ASSERT (!force || xd3_rlist_length (& stream->iopt_used) == 0); - - return 0; -} - -static int -xd3_iopt_get_slot (xd3_stream *stream, xd3_rinst** iptr) -{ - xd3_rinst *i; - int ret; - - if (xd3_rlist_empty (& stream->iopt_free)) - { - if (stream->iopt_unlimited) - { - int elts = XD3_ALLOCSIZE / sizeof(xd3_rinst); - - if ((ret = xd3_alloc_iopt (stream, elts))) - { - return ret; - } - - stream->iopt_size += elts; - } - else - { - if ((ret = xd3_iopt_flush_instructions (stream, 0))) { return ret; } - - XD3_ASSERT (! xd3_rlist_empty (& stream->iopt_free)); - } - } - - i = xd3_rlist_pop_back (& stream->iopt_free); - - xd3_rlist_push_back (& stream->iopt_used, i); - - (*iptr) = i; - - ++stream->i_slots_used; - - return 0; -} - -/* A copy is about to be emitted that extends backwards to POS, - * therefore it may completely cover some existing instructions in the - * buffer. If an instruction is completely covered by this new match, - * erase it. If the new instruction is covered by the previous one, - * return 1 to skip it. */ -static void -xd3_iopt_erase (xd3_stream *stream, usize_t pos, usize_t size) -{ - while (! xd3_rlist_empty (& stream->iopt_used)) - { - xd3_rinst *r = xd3_rlist_back (& stream->iopt_used); - - /* Verify that greedy is working. The previous instruction - * should end before the new one begins. */ - XD3_ASSERT ((stream->flags & XD3_BEGREEDY) == 0 || (r->pos + r->size <= pos)); - /* Verify that min_match is working. The previous instruction - * should end before the new one ends. */ - XD3_ASSERT ((stream->flags & XD3_BEGREEDY) != 0 || (r->pos + r->size < pos + size)); - - /* See if the last instruction starts before the new - * instruction. If so, there is nothing to erase. */ - if (r->pos < pos) - { - return; - } - - /* Otherwise, the new instruction covers the old one, delete it - and repeat. */ - xd3_rlist_remove (r); - xd3_rlist_push_back (& stream->iopt_free, r); - --stream->i_slots_used; - } -} - -/* This function tells the last matched input position. */ -static usize_t -xd3_iopt_last_matched (xd3_stream *stream) -{ - xd3_rinst *r; - - if (xd3_rlist_empty (& stream->iopt_used)) - { - return 0; - } - - r = xd3_rlist_back (& stream->iopt_used); - - return r->pos + r->size; -} - -/********************************************************* - Emit routines - ***********************************************************/ - -static int -xd3_emit_single (xd3_stream *stream, xd3_rinst *single, usize_t code) -{ - int has_size = stream->code_table[code].size1 == 0; - int ret; - - IF_DEBUG1 (DP(RINT "[emit1] %u %s (%u) code %u\n", - single->pos, - xd3_rtype_to_string ((xd3_rtype) single->type, 0), - single->size, - code)); - - if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) - { - return ret; - } - - if (has_size) - { - if ((ret = xd3_emit_size (stream, & INST_TAIL (stream), single->size))) - { - return ret; - } - } - - return 0; -} - -static int -xd3_emit_double (xd3_stream *stream, xd3_rinst *first, - xd3_rinst *second, usize_t code) -{ - int ret; - - /* All double instructions use fixed sizes, so all we need to do is - * output the instruction code, no sizes. */ - XD3_ASSERT (stream->code_table[code].size1 != 0 && - stream->code_table[code].size2 != 0); - - if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) - { - return ret; - } - - IF_DEBUG1 (DP(RINT "[emit2]: %u %s (%u) %s (%u) code %u\n", - first->pos, - xd3_rtype_to_string ((xd3_rtype) first->type, 0), - first->size, - xd3_rtype_to_string ((xd3_rtype) second->type, 0), - second->size, - code)); - - return 0; -} - -/* This enters a potential run instruction into the iopt buffer. The - * position argument is relative to the target window. */ -static int -xd3_emit_run (xd3_stream *stream, usize_t pos, usize_t size, uint8_t run_c) -{ - xd3_rinst* ri; - int ret; - - if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; } - - ri->type = XD3_RUN; - ri->xtra = run_c; - ri->pos = pos; - ri->size = size; - - return 0; -} - -/* This enters a potential copy instruction into the iopt buffer. The - * position argument is relative to the target window.. */ -int -xd3_found_match (xd3_stream *stream, usize_t pos, - usize_t size, xoff_t addr, int is_source) -{ - xd3_rinst* ri; - int ret; - - if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; } - - ri->type = XD3_CPY; - ri->xtra = is_source; - ri->pos = pos; - ri->size = size; - ri->addr = addr; - - return 0; -} - -static int -xd3_emit_hdr (xd3_stream *stream) -{ - int ret; - int use_secondary = stream->sec_type != NULL; - int use_adler32 = stream->flags & (XD3_ADLER32 | XD3_ADLER32_RECODE); - int vcd_source = xd3_encoder_used_source (stream); - usize_t win_ind = 0; - usize_t del_ind = 0; - usize_t enc_len; - usize_t tgt_len; - usize_t data_len; - usize_t inst_len; - usize_t addr_len; - - if (stream->current_window == 0) - { - usize_t hdr_ind = 0; - int use_appheader = stream->enc_appheader != NULL; - int use_gencodetbl = GENERIC_ENCODE_TABLES && - (stream->code_table_desc != & __rfc3284_code_table_desc); - - if (use_secondary) { hdr_ind |= VCD_SECONDARY; } - if (use_gencodetbl) { hdr_ind |= VCD_CODETABLE; } - if (use_appheader) { hdr_ind |= VCD_APPHEADER; } - - if ((ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_MAGIC1)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_MAGIC2)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_MAGIC3)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_VERSION)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), hdr_ind)) != 0) - { - return ret; - } - - /* Secondary compressor ID */ -#if SECONDARY_ANY - if (use_secondary && - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - stream->sec_type->id))) - { - return ret; - } -#endif - - /* Compressed code table */ - if (use_gencodetbl) - { - usize_t code_table_size; - const uint8_t *code_table_data; - - if ((ret = stream->comp_table_func (stream, & code_table_data, - & code_table_size))) - { - return ret; - } - - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), - code_table_size + 2)) || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - stream->code_table_desc->near_modes)) || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - stream->code_table_desc->same_modes)) || - (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), - code_table_data, code_table_size))) - { - return ret; - } - } - - /* Application header */ - if (use_appheader) - { - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), - stream->enc_appheadsz)) || - (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), - stream->enc_appheader, - stream->enc_appheadsz))) - { - return ret; - } - } - } - - /* try to compress this window */ -#if SECONDARY_ANY - if (use_secondary) - { - int data_sec = 0; - int inst_sec = 0; - int addr_sec = 0; - -# define ENCODE_SECONDARY_SECTION(UPPER,LOWER) \ - ((stream->flags & XD3_SEC_NO ## UPPER) == 0 && \ - (ret = xd3_encode_secondary (stream, \ - & UPPER ## _HEAD (stream), \ - & UPPER ## _TAIL (stream), \ - & xd3_sec_ ## LOWER (stream), \ - & stream->sec_ ## LOWER, \ - & LOWER ## _sec))) - - if (ENCODE_SECONDARY_SECTION (DATA, data) || - ENCODE_SECONDARY_SECTION (INST, inst) || - ENCODE_SECONDARY_SECTION (ADDR, addr)) - { - return ret; - } - - del_ind |= (data_sec ? VCD_DATACOMP : 0); - del_ind |= (inst_sec ? VCD_INSTCOMP : 0); - del_ind |= (addr_sec ? VCD_ADDRCOMP : 0); - } -#endif - - /* if (vcd_target) { win_ind |= VCD_TARGET; } */ - if (vcd_source) { win_ind |= VCD_SOURCE; } - if (use_adler32) { win_ind |= VCD_ADLER32; } - - /* window indicator */ - if ((ret = xd3_emit_byte (stream, & HDR_TAIL (stream), win_ind))) - { - return ret; - } - - /* source window */ - if (vcd_source) - { - /* or (vcd_target) { ... } */ - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), - stream->src->srclen)) || - (ret = xd3_emit_offset (stream, & HDR_TAIL (stream), - stream->src->srcbase))) { return ret; } - } - - tgt_len = stream->avail_in; - data_len = xd3_sizeof_output (DATA_HEAD (stream)); - inst_len = xd3_sizeof_output (INST_HEAD (stream)); - addr_len = xd3_sizeof_output (ADDR_HEAD (stream)); - - /* The enc_len field is a redundency for future extensions.*/ - enc_len = (1 + (xd3_sizeof_size (tgt_len) + - xd3_sizeof_size (data_len) + - xd3_sizeof_size (inst_len) + - xd3_sizeof_size (addr_len)) + - data_len + - inst_len + - addr_len + - (use_adler32 ? 4 : 0)); - - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), enc_len)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), tgt_len)) || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), del_ind)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), data_len)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), inst_len)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), addr_len))) - { - return ret; - } - - if (use_adler32) - { - uint8_t send[4]; - uint32_t a32; - - if (stream->flags & XD3_ADLER32) - { - a32 = adler32 (1L, stream->next_in, stream->avail_in); - } - else - { - a32 = stream->recode_adler32; - } - - send[0] = (a32 >> 24); - send[1] = (a32 >> 16); - send[2] = (a32 >> 8); - send[3] = (a32 & 0xff); - - if ((ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), send, 4))) - { - return ret; - } - } - - return 0; -} - -/**************************************************************** - Encode routines - ****************************************************************/ - -static int -xd3_encode_buffer_leftover (xd3_stream *stream) -{ - usize_t take; - usize_t room; - - /* Allocate the buffer. */ - if (stream->buf_in == NULL && - (stream->buf_in = (uint8_t*) xd3_alloc (stream, stream->winsize, 1)) == NULL) - { - return ENOMEM; - } - - /* Take leftover input first. */ - if (stream->buf_leftover != NULL) - { - XD3_ASSERT (stream->buf_avail == 0); - XD3_ASSERT (stream->buf_leftavail < stream->winsize); - - IF_DEBUG1 (DP(RINT "[leftover] previous %u avail %u\n", stream->buf_leftavail, stream->avail_in)); - - memcpy (stream->buf_in, stream->buf_leftover, stream->buf_leftavail); - - stream->buf_leftover = NULL; - stream->buf_avail = stream->buf_leftavail; - } - - /* Copy into the buffer. */ - room = stream->winsize - stream->buf_avail; - take = min (room, stream->avail_in); - - memcpy (stream->buf_in + stream->buf_avail, stream->next_in, take); - - stream->buf_avail += take; - - if (take < stream->avail_in) - { - /* Buffer is full */ - stream->buf_leftover = stream->next_in + take; - stream->buf_leftavail = stream->avail_in - take; - - IF_DEBUG1 (DP(RINT "[leftover] take %u remaining %u\n", take, stream->buf_leftavail)); - } - else if ((stream->buf_avail < stream->winsize) && !(stream->flags & XD3_FLUSH)) - { - /* Buffer has space */ - IF_DEBUG1 (DP(RINT "[leftover] %u emptied\n", take)); - return XD3_INPUT; - } - - /* Use the buffer: */ - stream->next_in = stream->buf_in; - stream->avail_in = stream->buf_avail; - stream->buf_avail = 0; - - return 0; -} - -/* Allocates one block of xd3_rlist elements */ -static int -xd3_alloc_iopt (xd3_stream *stream, int elts) -{ - int i; - xd3_iopt_buflist* last = - (xd3_iopt_buflist*) xd3_alloc (stream, sizeof (xd3_iopt_buflist), 1); - - if (last == NULL || - (last->buffer = (xd3_rinst*) xd3_alloc (stream, sizeof (xd3_rinst), elts)) == NULL) - { - return ENOMEM; - } - - last->next = stream->iopt_alloc; - stream->iopt_alloc = last; - - for (i = 0; i < elts; i += 1) - { - xd3_rlist_push_back (& stream->iopt_free, & last->buffer[i]); - } - - return 0; -} - -/* This function allocates all memory initially used by the encoder. */ -static int -xd3_encode_init (xd3_stream *stream, int full_init) -{ - int i; - - if (full_init) - { - int large_comp = (stream->src != NULL); - int small_comp = ! (stream->flags & XD3_NOCOMPRESS); - - /* Memory allocations for checksum tables are delayed until - * xd3_string_match_init in the first call to string_match--that way - * identical or short inputs require no table allocation. */ - if (large_comp) - { - usize_t hash_values = (stream->srcwin_maxsz / stream->smatcher.large_step); - - xd3_size_hashtable (stream, - hash_values, - & stream->large_hash); - } - - if (small_comp) - { - /* TODO: This is under devel: used to have min(sprevsz) here, which sort - * of makes sense, but observed fast performance w/ larger tables, which - * also sort of makes sense. @@@ */ - usize_t hash_values = stream->winsize; - - xd3_size_hashtable (stream, - hash_values, - & stream->small_hash); - } - } - - /* data buffers */ - for (i = 0; i < ENC_SECTS; i += 1) - { - if ((stream->enc_heads[i] = - stream->enc_tails[i] = - xd3_alloc_output (stream, NULL)) == NULL) - { - return ENOMEM; - } - } - - /* iopt buffer */ - xd3_rlist_init (& stream->iopt_used); - xd3_rlist_init (& stream->iopt_free); - - if (xd3_alloc_iopt (stream, stream->iopt_size) != 0) { goto fail; } - - XD3_ASSERT (xd3_rlist_length (& stream->iopt_free) == stream->iopt_size); - XD3_ASSERT (xd3_rlist_length (& stream->iopt_used) == 0); - - /* address cache, code table */ - stream->acache.s_near = stream->code_table_desc->near_modes; - stream->acache.s_same = stream->code_table_desc->same_modes; - stream->code_table = stream->code_table_func (); - - return xd3_alloc_cache (stream); - - fail: - - return ENOMEM; -} - -int -xd3_encode_init_full (xd3_stream *stream) -{ - return xd3_encode_init (stream, 1); -} - -int -xd3_encode_init_partial (xd3_stream *stream) -{ - return xd3_encode_init (stream, 0); -} - -/* Called after the ENC_POSTOUT state, this puts the output buffers - * back into separate lists and re-initializes some variables. (The - * output lists were spliced together during the ENC_FLUSH state.) */ -static void -xd3_encode_reset (xd3_stream *stream) -{ - int i; - xd3_output *olist; - - stream->avail_in = 0; - stream->small_reset = 1; - stream->i_slots_used = 0; - - if (stream->src != NULL) - { - stream->src->srcbase = 0; - stream->src->srclen = 0; - stream->srcwin_decided = 0; - stream->match_minaddr = 0; - stream->match_maxaddr = 0; - stream->taroff = 0; - } - - /* Reset output chains. */ - olist = stream->enc_heads[0]; - - for (i = 0; i < ENC_SECTS; i += 1) - { - XD3_ASSERT (olist != NULL); - - stream->enc_heads[i] = olist; - stream->enc_tails[i] = olist; - olist = olist->next_page; - - stream->enc_heads[i]->next = 0; - stream->enc_heads[i]->next_page = NULL; - - stream->enc_tails[i]->next_page = NULL; - stream->enc_tails[i] = stream->enc_heads[i]; - } - - xd3_freelist_output (stream, olist); -} - -/* The main encoding routine. */ -int -xd3_encode_input (xd3_stream *stream) -{ - int ret, i; - - if (stream->dec_state != 0) - { - stream->msg = "encoder/decoder transition"; - return XD3_INTERNAL; - } - - switch (stream->enc_state) - { - case ENC_INIT: - /* Only reached on first time through: memory setup. */ - if ((ret = xd3_encode_init_full (stream))) { return ret; } - - stream->enc_state = ENC_INPUT; - - case ENC_INPUT: - - /* If there is no input yet, just return. This checks for - * next_in == NULL, not avail_in == 0 since zero bytes is a - * valid input. There is an assertion in xd3_avail_input() that - * next_in != NULL for this reason. By returning right away we - * avoid creating an input buffer before the caller has supplied - * its first data. It is possible for xd3_avail_input to be - * called both before and after the first call to - * xd3_encode_input(). */ - if (stream->next_in == NULL) - { - return XD3_INPUT; - } - - enc_flush: - /* See if we should buffer the input: either if there is already - * a leftover buffer, or if the input is short of winsize - * without flush. The label at this point is reached by a goto - * below, when there is leftover input after postout. */ - if ((stream->buf_leftover != NULL) || - (stream->avail_in < stream->winsize && ! (stream->flags & XD3_FLUSH))) - { - if ((ret = xd3_encode_buffer_leftover (stream))) { return ret; } - } - - /* Initalize the address cache before each window. */ - xd3_init_cache (& stream->acache); - - stream->input_position = 0; - stream->min_match = MIN_MATCH; - stream->unencoded_offset = 0; - - stream->enc_state = ENC_SEARCH; - - IF_DEBUG1 (DP(RINT "[WINSTART:%"Q"u] input bytes %u offset %"Q"u\n", - stream->current_window, stream->avail_in, - stream->total_in)); - return XD3_WINSTART; - - case ENC_SEARCH: - - /* Reentrant matching. */ - if (stream->src != NULL) - { - switch (stream->match_state) - { - case MATCH_TARGET: - /* Try matching forward at the start of the target. - * This is entered the first time through, to check for - * a perfect match, and whenever there is a source match - * that extends to the end of the previous window. The - * match_srcpos field is initially zero and later set - * during xd3_source_extend_match. */ - - if (stream->avail_in > 0) - { - /* This call can't fail because the source window is - unrestricted. */ - ret = xd3_source_match_setup (stream, stream->match_srcpos); - XD3_ASSERT (ret == 0); - stream->match_state = MATCH_FORWARD; - } - else - { - stream->match_state = MATCH_SEARCHING; - stream->match_fwd = 0; - } - XD3_ASSERT (stream->match_fwd == 0); - - case MATCH_FORWARD: - case MATCH_BACKWARD: - if (stream->avail_in != 0) - { - if ((ret = xd3_source_extend_match (stream)) != 0) - { - return ret; - } - - /* The search has to make forward progress here - * or else it can get stuck in a match-backward - * (getsrcblk) then match-forward (getsrcblk), - * find insufficient match length, then repeat - * exactly the same search. - */ - stream->input_position += stream->match_fwd; - } - - case MATCH_SEARCHING: - /* Continue string matching. (It's possible that the - * initial match continued through the entire input, in - * which case we're still in MATCH_FORWARD and should - * remain so for the next input window.) */ - break; - } - } - - /* String matching... */ - if (stream->avail_in != 0 && - (ret = stream->smatcher.string_match (stream))) - { - return ret; - } - - stream->enc_state = ENC_INSTR; - - case ENC_INSTR: - /* Note: Jump here to encode VCDIFF deltas w/o using this - * string-matching code. */ - - /* Flush the instrution buffer, then possibly add one more - * instruction, then emit the header. */ - if ((ret = xd3_iopt_flush_instructions (stream, 1)) || - (ret = xd3_iopt_add_finalize (stream))) - { - return ret; - } - - stream->enc_state = ENC_FLUSH; - - case ENC_FLUSH: - /* Note: main_recode_func() bypasses string-matching by setting - * ENC_FLUSH. */ - if ((ret = xd3_emit_hdr (stream))) - { - return ret; - } - - /* Begin output. */ - stream->enc_current = HDR_HEAD (stream); - - /* Chain all the outputs together. After doing this, it looks - * as if there is only one section. The other enc_heads are set - * to NULL to avoid freeing them more than once. */ - for (i = 1; i < ENC_SECTS; i += 1) - { - stream->enc_tails[i-1]->next_page = stream->enc_heads[i]; - stream->enc_heads[i] = NULL; - } - - enc_output: - - stream->enc_state = ENC_POSTOUT; - stream->next_out = stream->enc_current->base; - stream->avail_out = stream->enc_current->next; - stream->total_out += (xoff_t) stream->avail_out; - - /* If there is any output in this buffer, return it, otherwise - * fall through to handle the next buffer or finish the window - * after all buffers have been output. */ - if (stream->avail_out > 0) - { - /* This is the only place xd3_encode returns XD3_OUTPUT */ - return XD3_OUTPUT; - } - - case ENC_POSTOUT: - - if (stream->avail_out != 0) - { - stream->msg = "missed call to consume output"; - return XD3_INTERNAL; - } - - /* Continue outputting one buffer at a time, until the next is NULL. */ - if ((stream->enc_current = stream->enc_current->next_page) != NULL) - { - goto enc_output; - } - - stream->total_in += (xoff_t) stream->avail_in; - stream->enc_state = ENC_POSTWIN; - - IF_DEBUG1 (DP(RINT "[WINFINISH:%"Q"u] in=%"Q"u\n", - stream->current_window, - stream->total_in)); - return XD3_WINFINISH; - - case ENC_POSTWIN: - - xd3_encode_reset (stream); - - stream->current_window += 1; - stream->enc_state = ENC_INPUT; - - /* If there is leftover input to flush, repeat. */ - if ((stream->buf_leftover != NULL) && (stream->flags & XD3_FLUSH)) - { - goto enc_flush; - } - - /* Ready for more input. */ - return XD3_INPUT; - - default: - stream->msg = "invalid state"; - return XD3_INTERNAL; - } -} -#endif /* XD3_ENCODER */ - -/***************************************************************** - Client convenience functions - ******************************************************************/ - -static int -xd3_process_stream (int is_encode, - xd3_stream *stream, - int (*func) (xd3_stream *), - int close_stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max) -{ - usize_t ipos = 0; - usize_t n = min(stream->winsize, input_size); - - (*output_size) = 0; - - stream->flags |= XD3_FLUSH; - - xd3_avail_input (stream, input + ipos, n); - ipos += n; - - for (;;) - { - int ret; - switch((ret = func (stream))) - { - case XD3_OUTPUT: { /* memcpy below */ break; } - case XD3_INPUT: { - n = min(stream->winsize, input_size - ipos); - if (n == 0) { - goto done; - } - xd3_avail_input (stream, input + ipos, n); - ipos += n; - continue; - } - case XD3_GOTHEADER: { /* ignore */ continue; } - case XD3_WINSTART: { /* ignore */ continue; } - case XD3_WINFINISH: { /* ignore */ continue; } - case XD3_GETSRCBLK: - { - stream->msg = "stream requires source input"; - return XD3_INTERNAL; - } - case 0: - { - /* xd3_encode_input/xd3_decode_input never return 0 */ - stream->msg = "invalid return: 0"; - return XD3_INTERNAL; - } - default: - return ret; - } - - if (*output_size + stream->avail_out > output_size_max) - { - stream->msg = "insufficient output space"; - return ENOSPC; - } - - memcpy (output + *output_size, stream->next_out, stream->avail_out); - - *output_size += stream->avail_out; - - xd3_consume_output (stream); - } - done: - return (close_stream == 0) ? 0 : xd3_close_stream (stream); -} - -static int -xd3_process_memory (int is_encode, - int (*func) (xd3_stream *), - int close_stream, - const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max, - int flags) { - xd3_stream stream; - xd3_config config; - xd3_source src; - int ret; - - memset (& stream, 0, sizeof (stream)); - memset (& config, 0, sizeof (config)); - - if (input == NULL || output == NULL) { - stream.msg = "invalid input/output buffer"; - ret = XD3_INTERNAL; - goto exit; - } - - config.flags = flags; - - if (is_encode) - { - config.srcwin_maxsz = source_size; - config.winsize = min(input_size, (usize_t) XD3_DEFAULT_WINSIZE); - config.iopt_size = min(input_size / 32, XD3_DEFAULT_IOPT_SIZE); - config.iopt_size = max(config.iopt_size, 128U); - config.sprevsz = xd3_pow2_roundup (config.winsize); - } - - if ((ret = xd3_config_stream (&stream, &config)) != 0) - { - goto exit; - } - - if (source != NULL) - { - memset (& src, 0, sizeof (src)); - src.size = source_size; - - src.blksize = source_size; - src.onblk = source_size; - src.curblk = source; - src.curblkno = 0; - - if ((ret = xd3_set_source (&stream, &src)) != 0) - { - goto exit; - } - } - - if ((ret = xd3_process_stream (is_encode, - & stream, - func, 1, - input, input_size, - output, - output_size, - output_size_max)) != 0) - { - goto exit; - } - - exit: - if (ret != 0) { - IF_DEBUG1 (DP(RINT "process_memory: %d: %s", ret, stream.msg)); - } - xd3_free_stream(&stream); - return ret; -} - -int -xd3_decode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max) -{ - return xd3_process_stream (0, stream, & xd3_decode_input, 1, - input, input_size, - output, output_size, output_size_max); -} - -int -xd3_decode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max, - int flags) { - return xd3_process_memory (0, & xd3_decode_input, 1, - input, input_size, - source, source_size, - output, output_size, output_size_max, - flags); -} - - -#if XD3_ENCODER -int -xd3_encode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max) -{ - return xd3_process_stream (1, stream, & xd3_encode_input, 1, - input, input_size, - output, output_size, output_size_max); -} - -int -xd3_encode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max, - int flags) { - return xd3_process_memory (1, & xd3_encode_input, 1, - input, input_size, - source, source_size, - output, output_size, output_size_max, - flags); -} -#endif - - -/************************************************************* - String matching helpers - *************************************************************/ - -#if XD3_ENCODER -/* Do the initial xd3_string_match() checksum table setup. - * Allocations are delayed until first use to avoid allocation - * sometimes (e.g., perfect matches, zero-length inputs). */ -static int -xd3_string_match_init (xd3_stream *stream) -{ - const int DO_SMALL = ! (stream->flags & XD3_NOCOMPRESS); - const int DO_LARGE = (stream->src != NULL); - - if (DO_LARGE && stream->large_table == NULL) - { - if ((stream->large_table = - (usize_t*) xd3_alloc0 (stream, stream->large_hash.size, sizeof (usize_t))) == NULL) - { - return ENOMEM; - } - } - - if (DO_SMALL) - { - /* Subsequent calls can return immediately after checking reset. */ - if (stream->small_table != NULL) - { - /* The target hash table is reinitialized once per window. */ - /* TODO: This would not have to be reinitialized if absolute - * offsets were being stored. */ - if (stream->small_reset) - { - stream->small_reset = 0; - memset (stream->small_table, 0, - sizeof (usize_t) * stream->small_hash.size); - } - - return 0; - } - - if ((stream->small_table = - (usize_t*) xd3_alloc0 (stream, - stream->small_hash.size, - sizeof (usize_t))) == NULL) - { - return ENOMEM; - } - - /* If there is a previous table needed. */ - if (stream->smatcher.small_lchain > 1 || - stream->smatcher.small_chain > 1) - { - if ((stream->small_prev = - (xd3_slist*) xd3_alloc (stream, - stream->sprevsz, - sizeof (xd3_slist))) == NULL) - { - return ENOMEM; - } - } - } - - return 0; -} - -#if XD3_USE_LARGEFILE64 -/* This function handles the 32/64bit ambiguity -- file positions are 64bit - * but the hash table for source-offsets is 32bit. */ -static xoff_t -xd3_source_cksum_offset(xd3_stream *stream, usize_t low) -{ - xoff_t scp = stream->srcwin_cksum_pos; - xoff_t s0 = scp >> 32; - - usize_t sr = (usize_t) scp; - - if (s0 == 0) { - return low; - } - - /* This should not be >= because srcwin_cksum_pos is the next - * position to index. */ - if (low > sr) { - return (--s0 << 32) | low; - } - - return (s0 << 32) | low; -} -#else -static xoff_t -xd3_source_cksum_offset(xd3_stream *stream, usize_t low) -{ - return (xoff_t) low; -} -#endif - -/* This function sets up the stream->src fields srcbase, srclen. The - * call is delayed until these values are needed to encode a copy - * address. At this point the decision has to be made. */ -static int -xd3_srcwin_setup (xd3_stream *stream) -{ - xd3_source *src = stream->src; - xoff_t length, x; - - /* Check the undecided state. */ - XD3_ASSERT (src->srclen == 0 && src->srcbase == 0); - - /* Avoid repeating this call. */ - stream->srcwin_decided = 1; - - /* If the stream is flushing, then the iopt buffer was able to - * contain the complete encoding. If no copies were issued no - * source window is actually needed. This prevents the VCDIFF - * header from including source base/len. xd3_emit_hdr checks for - * srclen == 0. */ - if (stream->enc_state == ENC_INSTR && stream->match_maxaddr == 0) - { - goto done; - } - - /* Check for overflow, srclen is usize_t - this can't happen unless - * XD3_DEFAULT_SRCBACK and related parameters are extreme - should - * use smaller windows. */ - length = stream->match_maxaddr - stream->match_minaddr; - - x = (xoff_t) USIZE_T_MAX; - if (length > x) - { - stream->msg = "source window length overflow (not 64bit)"; - return XD3_INTERNAL; - } - - /* If ENC_INSTR, then we know the exact source window to use because - * no more copies can be issued. */ - if (stream->enc_state == ENC_INSTR) - { - src->srcbase = stream->match_minaddr; - src->srclen = (usize_t) length; - XD3_ASSERT (src->srclen); - goto done; - } - - /* Otherwise, we have to make a guess. More copies may still be - * issued, but we have to decide the source window base and length - * now. */ - src->srcbase = stream->match_minaddr; - src->srclen = max ((usize_t) length, stream->avail_in + (stream->avail_in >> 2)); - if (src->size < src->srcbase + (xoff_t) src->srclen) - { - /* Could reduce srcbase, as well. */ - src->srclen = src->size - src->srcbase; - } - - XD3_ASSERT (src->srclen); - done: - /* Set the taroff. This convenience variable is used even when - stream->src == NULL. */ - stream->taroff = src->srclen; - return 0; -} - -/* Sets the bounding region for a newly discovered source match, prior - * to calling xd3_source_extend_match(). This sets the match_maxfwd, - * match_maxback variables. Note: srcpos is an absolute position - * (xoff_t) but the match_maxfwd, match_maxback variables are usize_t. - * Returns 0 if the setup succeeds, or 1 if the source position lies - * outside an already-decided srcbase/srclen window. */ -static int -xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos) -{ - xd3_source *src = stream->src; - usize_t greedy_or_not; - - stream->match_maxback = 0; - stream->match_maxfwd = 0; - stream->match_back = 0; - stream->match_fwd = 0; - - /* This avoids a non-blocking endless loop caused by scanning - * backwards across a block boundary, only to find not enough - * matching bytes to beat the current min_match due to a better lazy - * target match: the re-entry to xd3_string_match() repeats the same - * long match because the input position hasn't changed. TODO: if - * ever duplicates are added to the source hash table, this logic - * won't suffice to avoid loops. See testing/regtest.cc's - * TestNonBlockingProgress test! */ - if (srcpos != 0 && srcpos == stream->match_last_srcpos) - { - goto bad; - } - - /* Going backwards, the 1.5-pass algorithm allows some - * already-matched input may be covered by a longer source match. - * The greedy algorithm does not allow this. */ - if (stream->flags & XD3_BEGREEDY) - { - /* The greedy algorithm allows backward matching to the last - matched position. */ - greedy_or_not = xd3_iopt_last_matched (stream); - } - else - { - /* The 1.5-pass algorithm allows backward matching to go back as - * far as the unencoded offset, which is updated as instructions - * pass out of the iopt buffer. If this (default) is chosen, it - * means xd3_iopt_erase may be called to eliminate instructions - * when a covering source match is found. */ - greedy_or_not = stream->unencoded_offset; - } - - /* Backward target match limit. */ - XD3_ASSERT (stream->input_position >= greedy_or_not); - stream->match_maxback = stream->input_position - greedy_or_not; - - /* Forward target match limit. */ - XD3_ASSERT (stream->avail_in > stream->input_position); - stream->match_maxfwd = stream->avail_in - stream->input_position; - - /* Now we take the source position into account. It depends whether - * the srclen/srcbase have been decided yet. */ - if (stream->srcwin_decided == 0) - { - /* Unrestricted case: the match can cover the entire source, - * 0--src->size. We compare the usize_t - * match_maxfwd/match_maxback against the xoff_t - * src->size/srcpos values and take the min. */ - xoff_t srcavail; - - if (srcpos < (xoff_t) stream->match_maxback) - { - stream->match_maxback = srcpos; - } - - srcavail = src->size - srcpos; - if (srcavail < (xoff_t) stream->match_maxfwd) - { - stream->match_maxfwd = srcavail; - } - - goto good; - } - - /* Decided some source window. */ - XD3_ASSERT (src->srclen > 0); - - /* Restricted case: fail if the srcpos lies outside the source window */ - if ((srcpos < src->srcbase) || (srcpos > (src->srcbase + (xoff_t) src->srclen))) - { - goto bad; - } - else - { - usize_t srcavail; - - srcavail = (usize_t) (srcpos - src->srcbase); - if (srcavail < stream->match_maxback) - { - stream->match_maxback = srcavail; - } - - srcavail = (usize_t) (src->srcbase + (xoff_t) src->srclen - srcpos); - if (srcavail < stream->match_maxfwd) { - stream->match_maxfwd = srcavail; - } - - goto good; - } - - good: - stream->match_state = MATCH_BACKWARD; - stream->match_srcpos = srcpos; - stream->match_last_srcpos = srcpos; - return 0; - - bad: - stream->match_state = MATCH_SEARCHING; - return 1; -} - -/* This code is experimental, and I'm having trouble benchmarking - * it reliably. */ -#if 0 -static inline int -xd3_forward_match(const uint8_t *s1c, const uint8_t *s2c, size_t n) -{ - size_t i = 0; -#if UNALIGNED_OK - size_t nint = n / sizeof(int); - - if (nint >> 3) - { - size_t j = 0; - const int *s1 = (const int*)s1c; - const int *s2 = (const int*)s2c; - size_t nint_8 = nint - 8; - - while (i <= nint_8 && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++]) { } - - i = (i - 1) * sizeof(int); - } -#endif - - while (i < n && s1c[i] == s2c[i]) - { - i++; - } - return i; -} -#else -static inline usize_t -xd3_forward_match(const uint8_t *s1c, - const uint8_t *s2c, - usize_t n) { - usize_t i = 0; - while (i < n && s1c[i] == s2c[i]) - { - i++; - } - return i; -} -#endif - - -/* This function expands the source match backward and forward. It is - * reentrant, since xd3_getblk may return XD3_GETSRCBLK, so most - * variables are kept in xd3_stream. There are two callers of this - * function, the string_matching routine when a checksum match is - * discovered, and xd3_encode_input whenever a continuing (or initial) - * match is suspected. The two callers do different things with the - * input_position, thus this function leaves that variable untouched. - * If a match is taken the resulting stream->match_fwd is left - * non-zero. */ -static int -xd3_source_extend_match (xd3_stream *stream) -{ - int ret; - xd3_source *src = stream->src; - xoff_t matchoff; /* matchoff is the current right/left-boundary of - the source match being tested. */ - usize_t streamoff; /* streamoff is the current right/left-boundary - of the input match being tested. */ - xoff_t tryblk; /* tryblk, tryoff are the block, offset position - of matchoff */ - usize_t tryoff; - usize_t tryrem; /* tryrem is the number of matchable bytes */ - usize_t matched; - - XD3_ASSERT (src != NULL); - - /* Does it make sense to compute backward match AFTER forward match? */ - if (stream->match_state == MATCH_BACKWARD) - { - /* Note: this code is practically duplicated below, substituting - * match_fwd/match_back and direction. Consolidate? */ - matchoff = stream->match_srcpos - stream->match_back; - streamoff = stream->input_position - stream->match_back; - xd3_blksize_div (matchoff, src, &tryblk, &tryoff); - - /* this loops backward over source blocks */ - while (stream->match_back < stream->match_maxback) - { - /* see if we're backing across a source block boundary */ - if (tryoff == 0) - { - tryoff = src->blksize; - tryblk -= 1; - } - - if ((ret = xd3_getblk (stream, tryblk))) - { - /* if search went too far back, continue forward. */ - if (ret == XD3_TOOFARBACK) - { - break; - } - - /* could be a XD3_GETSRCBLK failure. */ - return ret; - } - - /* TODO: This code can be optimized similar to xd3_match_forward() */ - for (tryrem = min (tryoff, stream->match_maxback - - stream->match_back); - tryrem != 0; - tryrem -= 1, stream->match_back += 1) - { - if (src->curblk[tryoff-1] != stream->next_in[streamoff-1]) - { - goto doneback; - } - - tryoff -= 1; - streamoff -= 1; - } - } - - doneback: - stream->match_state = MATCH_FORWARD; - } - - XD3_ASSERT (stream->match_state == MATCH_FORWARD); - - matchoff = stream->match_srcpos + stream->match_fwd; - streamoff = stream->input_position + stream->match_fwd; - xd3_blksize_div (matchoff, src, & tryblk, & tryoff); - - /* Note: practically the same code as backwards case above: same comments */ - while (stream->match_fwd < stream->match_maxfwd) - { - if (tryoff == src->blksize) - { - tryoff = 0; - tryblk += 1; - } - - if ((ret = xd3_getblk (stream, tryblk))) - { - /* if search went too far back, continue forward. */ - if (ret == XD3_TOOFARBACK) - { - break; - } - - /* could be a XD3_GETSRCBLK failure. */ - return ret; - } - - tryrem = min(stream->match_maxfwd - stream->match_fwd, - src->blksize - tryoff); - - matched = xd3_forward_match(src->curblk + tryoff, - stream->next_in + streamoff, - tryrem); - tryoff += matched; - streamoff += matched; - stream->match_fwd += matched; - - if (tryrem != matched) - { - break; - } - } - - stream->match_state = MATCH_SEARCHING; - - /* If the match ends short of the last instruction end, we probably - * don't want it. There is the possibility that a copy ends short - * of the last copy but also goes further back, in which case we - * might want it. This code does not implement such: if so we would - * need more complicated xd3_iopt_erase logic. */ - if (stream->match_fwd < stream->min_match) - { - stream->match_fwd = 0; - } - else - { - usize_t total = stream->match_fwd + stream->match_back; - - /* Correct the variables to remove match_back from the equation. */ - usize_t target_position = stream->input_position - stream->match_back; - usize_t match_length = stream->match_back + stream->match_fwd; - xoff_t match_position = stream->match_srcpos - stream->match_back; - xoff_t match_end = stream->match_srcpos + stream->match_fwd; - - /* At this point we may have to erase any iopt-buffer - * instructions that are fully covered by a backward-extending - * copy. */ - if (stream->match_back > 0) - { - xd3_iopt_erase (stream, target_position, total); - } - - stream->match_back = 0; - - /* Update ranges. The first source match occurs with both - values set to 0. */ - if (stream->match_maxaddr == 0 || - match_position < stream->match_minaddr) - { - stream->match_minaddr = match_position; - } - - if (match_end > stream->match_maxaddr) - { - /* Note: per-window */ - stream->match_maxaddr = match_end; - } - - if (match_end > stream->maxsrcaddr) - { - /* Note: across windows */ - stream->maxsrcaddr = match_end; - } - - IF_DEBUG1 ({ - static int x = 0; - DP(RINT "[source match:%d] <inp %"Q"u %"Q"u> <src %"Q"u %"Q"u> (%s) [ %u bytes ]\n", - x++, - stream->total_in + target_position, - stream->total_in + target_position + match_length, - match_position, - match_position + match_length, - (stream->total_in + target_position == match_position) ? "same" : "diff", - match_length); - }); - - if ((ret = xd3_found_match (stream, - /* decoder position */ target_position, - /* length */ match_length, - /* address */ match_position, - /* is_source */ 1))) - { - return ret; - } - - /* If the match ends with the available input: */ - if (target_position + match_length == stream->avail_in) - { - /* Setup continuing match for the next window. */ - stream->match_state = MATCH_TARGET; - stream->match_srcpos = match_end; - } - } - - return 0; -} - -/* Update the small hash. Values in the small_table are offset by - * HASH_CKOFFSET (1) to distinguish empty buckets from real offsets. */ -static void -xd3_scksum_insert (xd3_stream *stream, - usize_t inx, - usize_t scksum, - usize_t pos) -{ - /* If we are maintaining previous duplicates. */ - if (stream->small_prev) - { - usize_t last_pos = stream->small_table[inx]; - xd3_slist *pos_list = & stream->small_prev[pos & stream->sprevmask]; - - /* Note last_pos is offset by HASH_CKOFFSET. */ - pos_list->last_pos = last_pos; - } - - /* Enter the new position into the hash bucket. */ - stream->small_table[inx] = pos + HASH_CKOFFSET; -} - -#if XD3_DEBUG -static int -xd3_check_smatch (const uint8_t *ref0, const uint8_t *inp0, - const uint8_t *inp_max, usize_t cmp_len) -{ - usize_t i; - - for (i = 0; i < cmp_len; i += 1) - { - XD3_ASSERT (ref0[i] == inp0[i]); - } - - if (inp0 + cmp_len < inp_max) - { - XD3_ASSERT (inp0[i] != ref0[i]); - } - - return 1; -} -#endif /* XD3_DEBUG */ - -/* When the hash table indicates a possible small string match, it - * calls this routine to find the best match. The first matching - * position is taken from the small_table, HASH_CKOFFSET is subtracted - * to get the actual position. After checking that match, if previous - * linked lists are in use (because stream->smatcher.small_chain > 1), - * previous matches are tested searching for the longest match. If - * (stream->min_match > MIN_MATCH) then a lazy match is in effect. - */ -static usize_t -xd3_smatch (xd3_stream *stream, - usize_t base, - usize_t scksum, - usize_t *match_offset) -{ - usize_t cmp_len; - usize_t match_length = 0; - usize_t chain = (stream->min_match == MIN_MATCH ? - stream->smatcher.small_chain : - stream->smatcher.small_lchain); - const uint8_t *inp_max = stream->next_in + stream->avail_in; - const uint8_t *inp; - const uint8_t *ref; - - SMALL_HASH_DEBUG1 (stream, stream->next_in + stream->input_position); - - XD3_ASSERT (stream->min_match + stream->input_position <= stream->avail_in); - - base -= HASH_CKOFFSET; - - again: - - IF_DEBUG2 (DP(RINT "smatch at base=%u inp=%u cksum=%u\n", base, - stream->input_position, scksum)); - - /* For small matches, we can always go to the end-of-input because - * the matching position must be less than the input position. */ - XD3_ASSERT (base < stream->input_position); - - ref = stream->next_in + base; - inp = stream->next_in + stream->input_position; - - SMALL_HASH_DEBUG2 (stream, ref); - - /* Expand potential match forward. */ - while (inp < inp_max && *inp == *ref) - { - ++inp; - ++ref; - } - - cmp_len = inp - (stream->next_in + stream->input_position); - - /* Verify correctness */ - XD3_ASSERT (xd3_check_smatch (stream->next_in + base, - stream->next_in + stream->input_position, - inp_max, cmp_len)); - - /* Update longest match */ - if (cmp_len > match_length) - { - ( match_length) = cmp_len; - (*match_offset) = base; - - /* Stop if we match the entire input or have a long_enough match. */ - if (inp == inp_max || cmp_len >= stream->smatcher.long_enough) - { - goto done; - } - } - - /* If we have not reached the chain limit, see if there is another - previous position. */ - while (--chain != 0) - { - /* Calculate the previous offset. */ - usize_t prev_pos = stream->small_prev[base & stream->sprevmask].last_pos; - usize_t diff_pos; - - if (prev_pos == 0) - { - break; - } - - prev_pos -= HASH_CKOFFSET; - - if (prev_pos > base) - { - break; - } - - base = prev_pos; - - XD3_ASSERT (stream->input_position > base); - diff_pos = stream->input_position - base; - - /* Stop searching if we go beyond sprevsz, since those entries - * are for unrelated checksum entries. */ - if (diff_pos & ~stream->sprevmask) - { - break; - } - - goto again; - } - - done: - /* Crude efficiency test: if the match is very short and very far back, it's - * unlikely to help, but the exact calculation requires knowing the state of - * the address cache and adjacent instructions, which we can't do here. - * Rather than encode a probably inefficient copy here and check it later - * (which complicates the code a lot), do this: - */ - if (match_length == 4 && stream->input_position - (*match_offset) >= 1<<14) - { - /* It probably takes >2 bytes to encode an address >= 2^14 from here */ - return 0; - } - if (match_length == 5 && stream->input_position - (*match_offset) >= 1<<21) - { - /* It probably takes >3 bytes to encode an address >= 2^21 from here */ - return 0; - } - - /* It's unlikely that a window is large enough for the (match_length == 6 && - * address >= 2^28) check */ - return match_length; -} - -#if XD3_DEBUG -static void -xd3_verify_small_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum) -{ - uint32_t state; - uint32_t cksum = xd3_scksum (&state, inp, stream->smatcher.small_look); - - XD3_ASSERT (cksum == x_cksum); -} - -static void -xd3_verify_large_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum) -{ - uint32_t cksum = xd3_lcksum (inp, stream->smatcher.large_look); - XD3_ASSERT (cksum == x_cksum); -} -static void -xd3_verify_run_state (xd3_stream *stream, - const uint8_t *inp, - int x_run_l, - uint8_t x_run_c) -{ - int slook = stream->smatcher.small_look; - uint8_t run_c; - int run_l = xd3_comprun (inp, slook, &run_c); - - XD3_ASSERT (run_l == 0 || run_c == x_run_c); - XD3_ASSERT (x_run_l > slook || run_l == x_run_l); -} -#endif /* XD3_DEBUG */ - -/* This function computes more source checksums to advance the window. - * Called at every entrance to the string-match loop and each time - * stream->input_position reaches the value returned as - * *next_move_point. NB: this is one of the most expensive functions - * in this code and also the most critical for good compression. - * - * TODO: really would like a good test for this logic. how? - * Update: testing/regtest.cc has some basic tests, more would be nice. - * TODO: optimize the inner loop - */ -static int -xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) -{ - xoff_t logical_input_cksum_pos; - - XD3_ASSERT(stream->srcwin_cksum_pos <= stream->src->size); - if (stream->srcwin_cksum_pos == stream->src->size) - { - *next_move_point = USIZE_T_MAX; - return 0; - } - - /* Begin by advancing at twice the input rate, up to half the - * maximum window size. */ - logical_input_cksum_pos = min((stream->total_in + stream->input_position) * 2, - (stream->total_in + stream->input_position) + - (stream->srcwin_maxsz / 2)); - - /* If srcwin_cksum_pos is already greater, wait until the difference - * is met. */ - if (stream->srcwin_cksum_pos > logical_input_cksum_pos) - { - *next_move_point = stream->input_position + - (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos); - return 0; - } - - /* A long match may have extended past srcwin_cksum_pos. Don't - * start checksumming already-matched source data. */ - if (stream->maxsrcaddr > stream->srcwin_cksum_pos) - { - stream->srcwin_cksum_pos = stream->maxsrcaddr; - } - - if (logical_input_cksum_pos < stream->srcwin_cksum_pos) - { - logical_input_cksum_pos = stream->srcwin_cksum_pos; - } - - /* Advance at least one source block. With the command-line - * defaults this means: - * - * if (src->size <= srcwin_maxsz), index the entire source at once - * using the position of the first non-match. This is good for - * small inputs, especially when the content may have moved anywhere - * in the file (e.g., tar files). - * - * if (src->size > srcwin_maxsz), index at least one block (which - * the command-line sets to 1/32 of srcwin_maxsz) ahead of the - * logical position. This is good for different reasons: when a - * long match spanning several source blocks is encountered, this - * avoids computing checksums for those blocks. If the data can - * move anywhere, this is bad. - */ - logical_input_cksum_pos += stream->src->blksize; - - IF_DEBUG1 (DP(RINT "[srcwin_move_point] T=%"Q"u S=%"Q"u/%"Q"u\n", - stream->total_in + stream->input_position, - stream->srcwin_cksum_pos, - logical_input_cksum_pos)); - - while (stream->srcwin_cksum_pos < logical_input_cksum_pos && - stream->srcwin_cksum_pos < stream->src->size) - { - xoff_t blkno; - xoff_t blkbaseoffset; - usize_t blkrem; - ssize_t oldpos; - ssize_t blkpos; - int ret; - xd3_blksize_div (stream->srcwin_cksum_pos, - stream->src, &blkno, &blkrem); - oldpos = blkrem; - blkpos = xd3_bytes_on_srcblk_fast (stream->src, blkno); - - if (oldpos + stream->smatcher.large_look > (usize_t) blkpos) - { - stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; - continue; - } - - if ((ret = xd3_getblk (stream, blkno))) - { - /* TOOFARBACK should never occur here, since we read forward. */ - if (ret == XD3_TOOFARBACK) - { - ret = XD3_INTERNAL; - } - return ret; - } - - /* This inserts checksums for the entire block, in reverse, - * starting from the end of the block. This logic does not test - * stream->srcwin_cksum_pos because it always advances it to the - * start of the next block. - * - * oldpos is the srcwin_cksum_pos within this block. blkpos is - * the number of bytes available. Each iteration inspects - * large_look bytes then steps back large_step bytes. The - * if-stmt above ensures at least one large_look of data. */ - blkpos -= stream->smatcher.large_look; - blkbaseoffset = stream->src->blksize * blkno; - - do - { - uint32_t cksum = xd3_lcksum (stream->src->curblk + blkpos, - stream->smatcher.large_look); - usize_t hval = xd3_checksum_hash (& stream->large_hash, cksum); - - stream->large_table[hval] = - (usize_t) (blkbaseoffset + - (xoff_t)(blkpos + HASH_CKOFFSET)); - - IF_DEBUG (stream->large_ckcnt += 1); - - blkpos -= stream->smatcher.large_step; - } - while (blkpos >= oldpos); - - stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; - } - - if (stream->srcwin_cksum_pos >= stream->src->size) - { - /* This invariant is needed for xd3_source_cksum_offset() */ - stream->srcwin_cksum_pos = stream->src->size; - *next_move_point = USIZE_T_MAX; - return 0; - } - - /* How long until this function should be called again. */ - XD3_ASSERT(stream->srcwin_cksum_pos >= logical_input_cksum_pos); - *next_move_point = stream->input_position + 1 + - (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos); - return 0; -} - -#endif /* XD3_ENCODER */ - -/******************************************************************** - TEMPLATE pass - *********************************************************************/ - -#endif /* __XDELTA3_C_INLINE_PASS__ */ -#ifdef __XDELTA3_C_TEMPLATE_PASS__ - -#if XD3_ENCODER - -/******************************************************************** - Templates - *******************************************************************/ - -/* Template macros */ -#define XD3_TEMPLATE(x) XD3_TEMPLATE2(x,TEMPLATE) -#define XD3_TEMPLATE2(x,n) XD3_TEMPLATE3(x,n) -#define XD3_TEMPLATE3(x,n) x ## n -#define XD3_STRINGIFY(x) XD3_STRINGIFY2(x) -#define XD3_STRINGIFY2(x) #x - -static int XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream); - -static const xd3_smatcher XD3_TEMPLATE(__smatcher_) = -{ - XD3_STRINGIFY(TEMPLATE), - XD3_TEMPLATE(xd3_string_match_), -#if SOFTCFG == 1 - 0, 0, 0, 0, 0, 0, 0 -#else - LLOOK, LSTEP, SLOOK, SCHAIN, SLCHAIN, MAXLAZY, LONGENOUGH -#endif -}; - -static int -XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) -{ - const int DO_SMALL = ! (stream->flags & XD3_NOCOMPRESS); - const int DO_LARGE = (stream->src != NULL); - const int DO_RUN = (1); - - const uint8_t *inp; - uint32_t scksum = 0; - uint32_t scksum_state; - uint32_t lcksum = 0; - usize_t sinx; - usize_t linx; - uint8_t run_c; - size_t run_l; - int ret; - usize_t match_length; - usize_t match_offset = 0; - usize_t next_move_point; - - /* If there will be no compression due to settings or short input, - * skip it entirely. */ - if (! (DO_SMALL || DO_LARGE || DO_RUN) || - stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; } - - if ((ret = xd3_string_match_init (stream))) { return ret; } - - /* The restartloop label is reached when the incremental loop state - * needs to be reset. */ - restartloop: - - /* If there is not enough input remaining for any kind of match, - skip it. */ - if (stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; } - - /* Now reset the incremental loop state: */ - - /* The min_match variable is updated to avoid matching the same lazy - * match over and over again. For example, if you find a (small) - * match of length 9 at one position, you will likely find a match - * of length 8 at the next position. */ - if (xd3_iopt_last_matched (stream) > stream->input_position) - { - stream->min_match = max(MIN_MATCH, - 1 + xd3_iopt_last_matched(stream) - - stream->input_position); - } - else - { - stream->min_match = MIN_MATCH; - } - - /* The current input byte. */ - inp = stream->next_in + stream->input_position; - - /* Small match state. */ - if (DO_SMALL) - { - scksum = xd3_scksum (&scksum_state, inp, SLOOK); - } - - /* Run state. */ - if (DO_RUN) - { - run_l = xd3_comprun (inp, SLOOK, & run_c); - } - - /* Large match state. We continue the loop even after not enough - * bytes for LLOOK remain, so always check stream->input_position in - * DO_LARGE code. */ - if (DO_LARGE && (stream->input_position + LLOOK <= stream->avail_in)) - { - /* Source window: next_move_point is the point that - * stream->input_position must reach before computing more - * source checksum. */ - if ((ret = xd3_srcwin_move_point (stream, & next_move_point))) - { - return ret; - } - - lcksum = xd3_lcksum (inp, LLOOK); - } - - /* TRYLAZYLEN: True if a certain length match should be followed by - * lazy search. This checks that LEN is shorter than MAXLAZY and - * that there is enough leftover data to consider lazy matching. - * "Enough" is set to 2 since the next match will start at the next - * offset, it must match two extra characters. */ -#define TRYLAZYLEN(LEN,POS,MAX) ((MAXLAZY) > 0 && (LEN) < (MAXLAZY) \ - && (POS) + (LEN) <= (MAX) - 2) - - /* HANDLELAZY: This statement is called each time an instruciton is - * emitted (three cases). If the instruction is large enough, the - * loop is restarted, otherwise lazy matching may ensue. */ -#define HANDLELAZY(mlen) \ - if (TRYLAZYLEN ((mlen), (stream->input_position), (stream->avail_in))) \ - { stream->min_match = (mlen) + LEAST_MATCH_INCR; goto updateone; } \ - else \ - { stream->input_position += (mlen); goto restartloop; } - - /* Now loop over one input byte at a time until a match is found... */ - for (;; inp += 1, stream->input_position += 1) - { - /* Now we try three kinds of string match in order of expense: - * run, large match, small match. */ - - /* Expand the start of a RUN. The test for (run_l == SLOOK) - * avoids repeating this check when we pass through a run area - * performing lazy matching. The run is only expanded once when - * the min_match is first reached. If lazy matching is - * performed, the run_l variable will remain inconsistent until - * the first non-running input character is reached, at which - * time the run_l may then again grow to SLOOK. */ - if (DO_RUN && run_l == SLOOK) - { - usize_t max_len = stream->avail_in - stream->input_position; - - IF_DEBUG (xd3_verify_run_state (stream, inp, run_l, run_c)); - - while (run_l < max_len && inp[run_l] == run_c) { run_l += 1; } - - /* Output a RUN instruction. */ - if (run_l >= stream->min_match && run_l >= MIN_RUN) - { - if ((ret = xd3_emit_run (stream, stream->input_position, - run_l, run_c))) { return ret; } - - HANDLELAZY (run_l); - } - } - - /* If there is enough input remaining. */ - if (DO_LARGE && (stream->input_position + LLOOK <= stream->avail_in)) - { - if ((stream->input_position >= next_move_point) && - (ret = xd3_srcwin_move_point (stream, & next_move_point))) - { - return ret; - } - - linx = xd3_checksum_hash (& stream->large_hash, lcksum); - - IF_DEBUG (xd3_verify_large_state (stream, inp, lcksum)); - - if (stream->large_table[linx] != 0) - { - /* the match_setup will fail if the source window has - * been decided and the match lies outside it. You - * could consider forcing a window at this point to - * permit a new source window. */ - xoff_t adj_offset = - xd3_source_cksum_offset(stream, - stream->large_table[linx] - - HASH_CKOFFSET); - if (xd3_source_match_setup (stream, adj_offset) == 0) - { - if ((ret = xd3_source_extend_match (stream))) - { - return ret; - } - - /* Update stream position. match_fwd is zero if no - * match. */ - if (stream->match_fwd > 0) - { - HANDLELAZY (stream->match_fwd); - } - } - } - } - - /* Small matches. */ - if (DO_SMALL) - { - sinx = xd3_checksum_hash (& stream->small_hash, scksum); - - /* Verify incremental state in debugging mode. */ - IF_DEBUG (xd3_verify_small_state (stream, inp, scksum)); - - /* Search for the longest match */ - if (stream->small_table[sinx] != 0) - { - match_length = xd3_smatch (stream, - stream->small_table[sinx], - scksum, - & match_offset); - } - else - { - match_length = 0; - } - - /* Insert a hash for this string. */ - xd3_scksum_insert (stream, sinx, scksum, stream->input_position); - - /* Maybe output a COPY instruction */ - if (match_length >= stream->min_match) - { - IF_DEBUG1 ({ - static int x = 0; - DP(RINT "[target match:%d] <inp %u %u> <cpy %u %u> " - "(-%d) [ %u bytes ]\n", - x++, - stream->input_position, - stream->input_position + match_length, - match_offset, - match_offset + match_length, - stream->input_position - match_offset, - match_length); - }); - - if ((ret = xd3_found_match (stream, - /* decoder position */ - stream->input_position, - /* length */ match_length, - /* address */ match_offset, - /* is_source */ 0))) - { - return ret; - } - - /* Copy instruction. */ - HANDLELAZY (match_length); - } - } - - /* The logic above prevents excess work during lazy matching by - * increasing min_match to avoid smaller matches. Each time we - * advance stream->input_position by one, the minimum match - * shortens as well. */ - if (stream->min_match > MIN_MATCH) - { - stream->min_match -= 1; - } - - updateone: - - /* See if there are no more incremental cksums to compute. */ - if (stream->input_position + SLOOK == stream->avail_in) - { - goto loopnomore; - } - - /* Compute next RUN, CKSUM */ - if (DO_RUN) { NEXTRUN (inp[SLOOK]); } - if (DO_SMALL) - { - scksum = xd3_small_cksum_update (&scksum_state, inp, SLOOK); - } - if (DO_LARGE && (stream->input_position + LLOOK < stream->avail_in)) - { - lcksum = xd3_large_cksum_update (lcksum, inp, LLOOK); - } - } - - loopnomore: - return 0; -} -#endif /* XD3_ENCODER */ -#endif /* __XDELTA3_C_TEMPLATE_PASS__ */ diff --git a/xdelta3.h b/xdelta3.h deleted file mode 100644 index 3aab8b6..0000000 --- a/xdelta3.h +++ /dev/null @@ -1,1310 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* To know more about Xdelta, start by reading xdelta3.c. If you are - * ready to use the API, continue reading here. There are two - * interfaces -- xd3_encode_input and xd3_decode_input -- plus a dozen - * or so related calls. This interface is styled after Zlib. */ - -#ifndef _XDELTA3_H_ -#define _XDELTA3_H_ - -#include <stdlib.h> -#include <string.h> -#include <sys/types.h> - -/****************************************************************/ - -/* Default configured value of stream->winsize. If the program - * supplies xd3_encode_input() with data smaller than winsize the - * stream will automatically buffer the input, otherwise the input - * buffer is used directly. - */ -#ifndef XD3_DEFAULT_WINSIZE -#define XD3_DEFAULT_WINSIZE (1U << 23) -#endif - -/* Default total size of the source window used in xdelta3-main.h */ -#ifndef XD3_DEFAULT_SRCWINSZ -#define XD3_DEFAULT_SRCWINSZ (1U << 26) -#endif - -/* When Xdelta requests a memory allocation for certain buffers, it - * rounds up to units of at least this size. The code assumes (and - * asserts) that this is a power-of-two. */ -#ifndef XD3_ALLOCSIZE -#define XD3_ALLOCSIZE (1U<<14) -#endif - -/* The XD3_HARDMAXWINSIZE parameter is a safety mechanism to protect - * decoders against malicious files. The decoder will never decode a - * window larger than this. If the file specifies VCD_TARGET the - * decoder may require two buffers of this size. - * - * 8-16MB is reasonable, probably don't need to go larger. */ -#ifndef XD3_HARDMAXWINSIZE -#define XD3_HARDMAXWINSIZE (1U<<24) -#endif -/* The IOPT_SIZE value sets the size of a buffer used to batch - * overlapping copy instructions before they are optimized by picking - * the best non-overlapping ranges. The larger this buffer, the - * longer a forced xd3_srcwin_setup() decision is held off. Setting - * this value to 0 causes an unlimited buffer to be used. */ -#ifndef XD3_DEFAULT_IOPT_SIZE -#define XD3_DEFAULT_IOPT_SIZE (1U<<15) -#endif - -/* The maximum distance backward to search for small matches */ -#ifndef XD3_DEFAULT_SPREVSZ -#define XD3_DEFAULT_SPREVSZ (1U<<18) -#endif - -/* The default compression level - */ -#ifndef XD3_DEFAULT_LEVEL -#define XD3_DEFAULT_LEVEL 3 -#endif - -#ifndef XD3_DEFAULT_SECONDARY_LEVEL -#define XD3_DEFAULT_SECONDARY_LEVEL 6 -#endif - -#ifndef XD3_USE_LARGEFILE64 -#define XD3_USE_LARGEFILE64 1 -#endif - -/* Sizes and addresses within VCDIFF windows are represented as usize_t - * - * For source-file offsets and total file sizes, total input and - * output counts, the xoff_t type is used. The decoder and encoder - * generally check for overflow of the xoff_t size (this is tested at - * the 32bit boundary [xdelta3-test.h]). - */ -#ifndef _WIN32 -#include <stdint.h> -typedef unsigned int usize_t; -#else -#define WIN32_LEAN_AND_MEAN -#if XD3_USE_LARGEFILE64 -/* 64 bit file offsets: uses GetFileSizeEx and SetFilePointerEx. - * requires Win2000 or newer version of WinNT */ -#define WINVER 0x0500 -#define _WIN32_WINNT 0x0500 -#else -/* 32 bit (DWORD) file offsets: uses GetFileSize and - * SetFilePointer. compatible with win9x-me and WinNT4 */ -#define WINVER 0x0400 -#define _WIN32_WINNT 0x0400 -#endif -#include <windows.h> -typedef unsigned int usize_t; -#ifdef _MSC_VER -#define inline -typedef signed int ssize_t; -typedef unsigned char uint8_t; -typedef unsigned short uint16_t; -typedef unsigned long uint32_t; -typedef ULONGLONG uint64_t; -#else -/* mingw32, lcc and watcom provide a proper header */ -#include <stdint.h> -#endif -#endif - -/* TODO: note that SIZEOF_USIZE_T is never set to 8, although it should be for - * a 64bit platform. OTOH, may be that using 32bits is appropriate even on a - * 64bit platform because we allocate large arrays of these values. */ -#if XD3_USE_LARGEFILE64 -#define __USE_FILE_OFFSET64 1 /* GLIBC: for 64bit fileops, ... ? */ -typedef uint64_t xoff_t; -#define SIZEOF_XOFF_T 8 -#define SIZEOF_USIZE_T 4 -#ifndef WIN32 -#define Q "ll" -#else -#define Q "I64" -#endif -#else -typedef uint32_t xoff_t; -#define SIZEOF_XOFF_T 4 -#define SIZEOF_USIZE_T 4 -#define Q -#endif - -#define USE_UINT32 (SIZEOF_USIZE_T == 4 || \ - SIZEOF_XOFF_T == 4 || REGRESSION_TEST) -#define USE_UINT64 (SIZEOF_USIZE_T == 8 || \ - SIZEOF_XOFF_T == 8 || REGRESSION_TEST) - -/* TODO: probably should do something better here. */ -#ifndef UNALIGNED_OK -#if defined(__i386__) || defined(__i486__) || defined(__i586__) || \ - defined(__i686__) || defined(_X86_) || defined(__x86_64__) -#define UNALIGNED_OK 1 -#else -#define UNALIGNED_OK 0 -#endif -#endif - -/**********************************************************************/ - -/* Whether to build the encoder, otherwise only build the decoder. */ -#ifndef XD3_ENCODER -#define XD3_ENCODER 1 -#endif - -/* The code returned when main() fails, also defined in system - includes. */ -#ifndef EXIT_FAILURE -#define EXIT_FAILURE 1 -#endif - -/* REGRESSION TEST enables the "xdelta3 test" command, which runs a - series of self-tests. */ -#ifndef REGRESSION_TEST -#define REGRESSION_TEST 0 -#endif - -/* XD3_DEBUG=1 enables assertions and various statistics. Levels > 1 - * enable some additional output only useful during development and - * debugging. */ -#ifndef XD3_DEBUG -#define XD3_DEBUG 0 -#endif - -#ifndef PYTHON_MODULE -#define PYTHON_MODULE 0 -#endif - -#ifndef SWIG_MODULE -#define SWIG_MODULE 0 -#endif - -/* There are three string matching functions supplied: one fast, one - * slow (default), and one soft-configurable. To disable any of - * these, use the following definitions. */ -#ifndef XD3_BUILD_SLOW -#define XD3_BUILD_SLOW 1 -#endif -#ifndef XD3_BUILD_FAST -#define XD3_BUILD_FAST 1 -#endif -#ifndef XD3_BUILD_FASTER -#define XD3_BUILD_FASTER 1 -#endif -#ifndef XD3_BUILD_FASTEST -#define XD3_BUILD_FASTEST 1 -#endif -#ifndef XD3_BUILD_SOFT -#define XD3_BUILD_SOFT 1 -#endif -#ifndef XD3_BUILD_DEFAULT -#define XD3_BUILD_DEFAULT 1 -#endif - -#if XD3_DEBUG -#include <stdio.h> -#endif - -/* XPRINT. Debug output and VCDIFF_TOOLS functions report to stderr. - * I have used an irregular style to abbreviate [fprintf(stderr, "] as - * [DP(RINT "]. */ -#define DP fprintf -#define RINT stderr, - -typedef struct _xd3_stream xd3_stream; -typedef struct _xd3_source xd3_source; -typedef struct _xd3_hash_cfg xd3_hash_cfg; -typedef struct _xd3_smatcher xd3_smatcher; -typedef struct _xd3_rinst xd3_rinst; -typedef struct _xd3_dinst xd3_dinst; -typedef struct _xd3_hinst xd3_hinst; -typedef struct _xd3_winst xd3_winst; -typedef struct _xd3_rpage xd3_rpage; -typedef struct _xd3_addr_cache xd3_addr_cache; -typedef struct _xd3_output xd3_output; -typedef struct _xd3_desect xd3_desect; -typedef struct _xd3_iopt_buflist xd3_iopt_buflist; -typedef struct _xd3_rlist xd3_rlist; -typedef struct _xd3_sec_type xd3_sec_type; -typedef struct _xd3_sec_cfg xd3_sec_cfg; -typedef struct _xd3_sec_stream xd3_sec_stream; -typedef struct _xd3_config xd3_config; -typedef struct _xd3_code_table_desc xd3_code_table_desc; -typedef struct _xd3_code_table_sizes xd3_code_table_sizes; -typedef struct _xd3_slist xd3_slist; -typedef struct _xd3_whole_state xd3_whole_state; -typedef struct _xd3_wininfo xd3_wininfo; - -/* The stream configuration has three callbacks functions, all of - * which may be supplied with NULL values. If config->getblk is - * provided as NULL, the stream returns XD3_GETSRCBLK. */ - -typedef void* (xd3_alloc_func) (void *opaque, - usize_t items, - usize_t size); -typedef void (xd3_free_func) (void *opaque, - void *address); - -typedef int (xd3_getblk_func) (xd3_stream *stream, - xd3_source *source, - xoff_t blkno); - -/* These are internal functions to delay construction of encoding - * tables and support alternate code tables. See the comments & code - * enabled by GENERIC_ENCODE_TABLES. */ - -typedef const xd3_dinst* (xd3_code_table_func) (void); -typedef int (xd3_comp_table_func) (xd3_stream *stream, - const uint8_t **data, - usize_t *size); - - - -#if XD3_DEBUG -#define XD3_ASSERT(x) \ - do { if (! (x)) { DP(RINT "%s:%d: XD3 assertion failed: %s\n", __FILE__, __LINE__, #x); \ - abort (); } } while (0) -#else -#define XD3_ASSERT(x) (void)0 -#endif - -#ifdef __GNUC__ -/* As seen on linux-kernel. */ -#ifndef max -#define max(x,y) ({ \ - const typeof(x) _x = (x); \ - const typeof(y) _y = (y); \ - (void) (&_x == &_y); \ - _x > _y ? _x : _y; }) -#endif - -#ifndef min -#define min(x,y) ({ \ - const typeof(x) _x = (x); \ - const typeof(y) _y = (y); \ - (void) (&_x == &_y); \ - _x < _y ? _x : _y; }) -#endif -#else -#ifndef max -#define max(x,y) ((x) < (y) ? (y) : (x)) -#endif -#ifndef min -#define min(x,y) ((x) < (y) ? (x) : (y)) -#endif -#endif - -/**************************************************************** - PUBLIC ENUMS - ******************************************************************/ - -/* These are the five ordinary status codes returned by the - * xd3_encode_input() and xd3_decode_input() state machines. */ -typedef enum { - - /* An application must be prepared to handle these five return - * values from either xd3_encode_input or xd3_decode_input, except - * in the case of no-source compression, in which case XD3_GETSRCBLK - * is never returned. More detailed comments for these are given in - * xd3_encode_input and xd3_decode_input comments, below. */ - XD3_INPUT = -17703, /* need input */ - XD3_OUTPUT = -17704, /* have output */ - XD3_GETSRCBLK = -17705, /* need a block of source input (with no - * xd3_getblk function), a chance to do - * non-blocking read. */ - XD3_GOTHEADER = -17706, /* (decode-only) after the initial VCDIFF & - first window header */ - XD3_WINSTART = -17707, /* notification: returned before a window is - * processed, giving a chance to - * XD3_SKIP_WINDOW or not XD3_SKIP_EMIT that - * window. */ - XD3_WINFINISH = -17708, /* notification: returned after - encode/decode & output for a window */ - XD3_TOOFARBACK = -17709, /* (encoder only) may be returned by - getblk() if the block is too old */ - XD3_INTERNAL = -17710, /* internal error */ - XD3_INVALID = -17711, /* invalid config */ - XD3_INVALID_INPUT = -17712, /* invalid input/decoder error */ - XD3_NOSECOND = -17713, /* when secondary compression finds no - improvement. */ - -} xd3_rvalues; - -/* special values in config->flags */ -typedef enum -{ - XD3_JUST_HDR = (1 << 1), /* used by VCDIFF tools, see - xdelta3-main.h. */ - XD3_SKIP_WINDOW = (1 << 2), /* used by VCDIFF tools, see - xdelta3-main.h. */ - XD3_SKIP_EMIT = (1 << 3), /* used by VCDIFF tools, see - xdelta3-main.h. */ - XD3_FLUSH = (1 << 4), /* flush the stream buffer to - prepare for - xd3_stream_close(). */ - - XD3_SEC_DJW = (1 << 5), /* use DJW static huffman */ - XD3_SEC_FGK = (1 << 6), /* use FGK adaptive huffman */ - XD3_SEC_TYPE = (XD3_SEC_DJW | XD3_SEC_FGK), - - XD3_SEC_NODATA = (1 << 7), /* disable secondary compression of - the data section. */ - XD3_SEC_NOINST = (1 << 8), /* disable secondary compression of - the inst section. */ - XD3_SEC_NOADDR = (1 << 9), /* disable secondary compression of - the addr section. */ - - XD3_SEC_NOALL = (XD3_SEC_NODATA | XD3_SEC_NOINST | XD3_SEC_NOADDR), - - XD3_ADLER32 = (1 << 10), /* enable checksum computation in - the encoder. */ - XD3_ADLER32_NOVER = (1 << 11), /* disable checksum verification in - the decoder. */ - - XD3_ALT_CODE_TABLE = (1 << 12), /* for testing th - e alternate code table encoding. */ - - XD3_NOCOMPRESS = (1 << 13), /* disable ordinary data - * compression feature, only search - * the source, not the target. */ - XD3_BEGREEDY = (1 << 14), /* disable the "1.5-pass - * algorithm", instead use greedy - * matching. Greedy is off by - * default. */ - XD3_ADLER32_RECODE = (1 << 15), /* used by "recode". */ - - /* 4 bits to set the compression level the same as the command-line - * setting -1 through -9 (-0 corresponds to the XD3_NOCOMPRESS flag, - * and is independent of compression level). This is for - * convenience, especially with xd3_encode_memory(). */ - - XD3_COMPLEVEL_SHIFT = 20, /* 20 - 24 */ - XD3_COMPLEVEL_MASK = (0xF << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_1 = (1 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_2 = (2 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_3 = (3 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_6 = (6 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_9 = (9 << XD3_COMPLEVEL_SHIFT), - -} xd3_flags; - -/* The values of this enumeration are set in xd3_config using the - * smatch_cfg variable. It can be set to default, slow, fast, etc., - * and soft. */ -typedef enum -{ - XD3_SMATCH_DEFAULT = 0, /* Flags may contain XD3_COMPLEVEL bits, - else default. */ - XD3_SMATCH_SLOW = 1, - XD3_SMATCH_FAST = 2, - XD3_SMATCH_FASTER = 3, - XD3_SMATCH_FASTEST = 4, - XD3_SMATCH_SOFT = 5, -} xd3_smatch_cfg; - -/********************************************************************* - PRIVATE ENUMS -**********************************************************************/ - -/* stream->match_state is part of the xd3_encode_input state machine - * for source matching: - * - * 1. the XD3_GETSRCBLK block-read mechanism means reentrant matching - * 2. this state spans encoder windows: a match and end-of-window - * will continue in the next 3. the initial target byte and source - * byte are a presumed match, to avoid some computation in case the - * inputs are identical. - */ -typedef enum { - - MATCH_TARGET = 0, /* in this state, attempt to match the start of - * the target with the previously set source - * address (initially 0). */ - MATCH_BACKWARD = 1, /* currently expanding a match backward in the - source/target. */ - MATCH_FORWARD = 2, /* currently expanding a match forward in the - source/target. */ - MATCH_SEARCHING = 3, /* currently searching for a match. */ - -} xd3_match_state; - -/* The xd3_encode_input state machine steps through these states in - * the following order. The matcher is reentrant and returns - * XD3_INPUT whenever it requires more data. After receiving - * XD3_INPUT, if the application reads EOF it should call - * xd3_stream_close(). - */ -typedef enum { - - ENC_INIT = 0, /* xd3_encode_input has never been called. */ - ENC_INPUT = 1, /* waiting for xd3_avail_input () to be called. */ - ENC_SEARCH = 2, /* currently searching for matches. */ - ENC_INSTR = 3, /* currently formatting output. */ - ENC_FLUSH = 4, /* currently emitting output. */ - ENC_POSTOUT = 5, /* after an output section. */ - ENC_POSTWIN = 6, /* after all output sections. */ - ENC_ABORTED = 7, /* abort. */ -} xd3_encode_state; - -/* The xd3_decode_input state machine steps through these states in - * the following order. The matcher is reentrant and returns - * XD3_INPUT whenever it requires more data. After receiving - * XD3_INPUT, if the application reads EOF it should call - * xd3_stream_close(). - * - * 0-8: the VCDIFF header - * 9-18: the VCDIFF window header - * 19-21: the three primary sections: data, inst, addr - * 22: producing output: returns XD3_OUTPUT, possibly XD3_GETSRCBLK, - * 23: return XD3_WINFINISH, set state=9 to decode more input - */ -typedef enum { - - DEC_VCHEAD = 0, /* VCDIFF header */ - DEC_HDRIND = 1, /* header indicator */ - - DEC_SECONDID = 2, /* secondary compressor ID */ - - DEC_TABLEN = 3, /* code table length */ - DEC_NEAR = 4, /* code table near */ - DEC_SAME = 5, /* code table same */ - DEC_TABDAT = 6, /* code table data */ - - DEC_APPLEN = 7, /* application data length */ - DEC_APPDAT = 8, /* application data */ - - DEC_WININD = 9, /* window indicator */ - - DEC_CPYLEN = 10, /* copy window length */ - DEC_CPYOFF = 11, /* copy window offset */ - - DEC_ENCLEN = 12, /* length of delta encoding */ - DEC_TGTLEN = 13, /* length of target window */ - DEC_DELIND = 14, /* delta indicator */ - - DEC_DATALEN = 15, /* length of ADD+RUN data */ - DEC_INSTLEN = 16, /* length of instruction data */ - DEC_ADDRLEN = 17, /* length of address data */ - - DEC_CKSUM = 18, /* window checksum */ - - DEC_DATA = 19, /* data section */ - DEC_INST = 20, /* instruction section */ - DEC_ADDR = 21, /* address section */ - - DEC_EMIT = 22, /* producing data */ - - DEC_FINISH = 23, /* window finished */ - - DEC_ABORTED = 24, /* xd3_abort_stream */ -} xd3_decode_state; - -/************************************************************ - internal types - ************************************************************/ - -/* instruction lists used in the IOPT buffer */ -struct _xd3_rlist -{ - xd3_rlist *next; - xd3_rlist *prev; -}; - -/* the raw encoding of an instruction used in the IOPT buffer */ -struct _xd3_rinst -{ - uint8_t type; - uint8_t xtra; - uint8_t code1; - uint8_t code2; - usize_t pos; - usize_t size; - xoff_t addr; - xd3_rlist link; -}; - -/* the code-table form of an single- or double-instruction */ -struct _xd3_dinst -{ - uint8_t type1; - uint8_t size1; - uint8_t type2; - uint8_t size2; -}; - -/* the decoded form of a single (half) instruction. */ -struct _xd3_hinst -{ - uint8_t type; - uint32_t size; /* TODO: why decode breaks if this is usize_t? */ - uint32_t addr; /* TODO: why decode breaks if this is usize_t? */ -}; - -/* the form of a whole-file instruction */ -struct _xd3_winst -{ - uint8_t type; /* RUN, ADD, COPY */ - uint8_t mode; /* 0, VCD_SOURCE, VCD_TARGET */ - usize_t size; - xoff_t addr; - xoff_t position; /* absolute position of this inst */ -}; - -/* used by the encoder to buffer output in sections. list of blocks. */ -struct _xd3_output -{ - uint8_t *base; - usize_t next; - usize_t avail; - xd3_output *next_page; -}; - -/* used by the decoder to buffer input in sections. */ -struct _xd3_desect -{ - const uint8_t *buf; - const uint8_t *buf_max; - uint32_t size; /* TODO: why decode breaks if this is usize_t? */ - usize_t pos; - - /* used in xdelta3-decode.h */ - uint8_t *copied1; - usize_t alloc1; - - /* used in xdelta3-second.h */ - uint8_t *copied2; - usize_t alloc2; -}; - -/* the VCDIFF address cache, see the RFC */ -struct _xd3_addr_cache -{ - usize_t s_near; - usize_t s_same; - usize_t next_slot; /* the circular index for near */ - usize_t *near_array; /* array of size s_near */ - usize_t *same_array; /* array of size s_same*256 */ -}; - -/* the IOPT buffer list is just a list of buffers, which may be allocated - * during encode when using an unlimited buffer. */ -struct _xd3_iopt_buflist -{ - xd3_rinst *buffer; - xd3_iopt_buflist *next; -}; - -/* This is the record of a pre-compiled configuration, a subset of - xd3_config. */ -struct _xd3_smatcher -{ - const char *name; - int (*string_match) (xd3_stream *stream); - usize_t large_look; - usize_t large_step; - usize_t small_look; - usize_t small_chain; - usize_t small_lchain; - usize_t max_lazy; - usize_t long_enough; -}; - -/* hash table size & power-of-two hash function. */ -struct _xd3_hash_cfg -{ - usize_t size; - usize_t shift; - usize_t mask; -}; - -/* the sprev list */ -struct _xd3_slist -{ - usize_t last_pos; -}; - -/* window info (for whole state) */ -struct _xd3_wininfo { - xoff_t offset; - usize_t length; - uint32_t adler32; -}; - -/* whole state for, e.g., merge */ -struct _xd3_whole_state { - usize_t addslen; - uint8_t *adds; - usize_t adds_alloc; - - usize_t instlen; - xd3_winst *inst; - usize_t inst_alloc; - - usize_t wininfolen; - xd3_wininfo *wininfo; - usize_t wininfo_alloc; - - xoff_t length; -}; - -/******************************************************************** - public types - *******************************************************************/ - -/* Settings for the secondary compressor. */ -struct _xd3_sec_cfg -{ - int data_type; /* Which section. (set automatically) */ - int ngroups; /* Number of DJW Huffman groups. */ - int sector_size; /* Sector size. */ - int inefficient; /* If true, ignore efficiency check [avoid XD3_NOSECOND]. */ -}; - -/* This is the user-visible stream configuration. */ -struct _xd3_config -{ - usize_t winsize; /* The encoder window size. */ - usize_t sprevsz; /* How far back small string - matching goes */ - usize_t iopt_size; /* entries in the - instruction-optimizing - buffer */ - usize_t srcwin_maxsz; /* srcwin_size grows by a factor - of 2 when no matches are - found */ - - xd3_getblk_func *getblk; /* The three callbacks. */ - xd3_alloc_func *alloc; - xd3_free_func *freef; - void *opaque; /* Not used. */ - int flags; /* stream->flags are initialized - * from xd3_config & never - * modified by the library. Use - * xd3_set_flags to modify flags - * settings mid-stream. */ - - xd3_sec_cfg sec_data; /* Secondary compressor config: data */ - xd3_sec_cfg sec_inst; /* Secondary compressor config: inst */ - xd3_sec_cfg sec_addr; /* Secondary compressor config: addr */ - - xd3_smatch_cfg smatch_cfg; /* See enum: use fields below for - soft config */ - xd3_smatcher smatcher_soft; -}; - -/* The primary source file object. You create one of these objects and - * initialize the first four fields. This library maintains the next - * 5 fields. The configured getblk implementation is responsible for - * setting the final 3 fields when called (and/or when XD3_GETSRCBLK - * is returned). - */ -struct _xd3_source -{ - /* you set */ - xoff_t size; /* size of this source */ - usize_t blksize; /* block size */ - const char *name; /* its name, for debug/print - purposes */ - void *ioh; /* opaque handle */ - - /* getblk sets */ - xoff_t curblkno; /* current block number: client - sets after getblk request */ - usize_t onblk; /* number of bytes on current - block: client sets, xd3 - verifies */ - const uint8_t *curblk; /* current block array: client - sets after getblk request */ - - /* xd3 sets */ - usize_t srclen; /* length of this source window */ - xoff_t srcbase; /* offset of this source window - in the source itself */ - xoff_t blocks; /* the total number of blocks in - this source */ - usize_t onlastblk; /* cached size info, avoid __udivdi3 */ - int shiftby; /* for power-of-two blocksizes */ - int maskby; /* for power-of-two blocksizes */ - xoff_t cpyoff_blocks; /* offset of dec_cpyoff in blocks */ - usize_t cpyoff_blkoff; /* offset of copy window in - blocks, remainder */ - xoff_t getblkno; /* request block number: xd3 sets - current getblk request */ -}; - -/* The primary xd3_stream object, used for encoding and decoding. You - * may access only two fields: avail_out, next_out. Use the methods - * above to operate on xd3_stream. */ -struct _xd3_stream -{ - /* input state */ - const uint8_t *next_in; /* next input byte */ - usize_t avail_in; /* number of bytes available at - next_in */ - xoff_t total_in; /* how many bytes in */ - - /* output state */ - uint8_t *next_out; /* next output byte */ - usize_t avail_out; /* number of bytes available at - next_out */ - usize_t space_out; /* total out space */ - xoff_t current_window; /* number of windows encoded/decoded */ - xoff_t total_out; /* how many bytes out */ - - /* to indicate an error, xd3 sets */ - const char *msg; /* last error message, NULL if - no error */ - - /* source configuration */ - xd3_source *src; /* source array */ - - /* encoder memory configuration */ - usize_t winsize; /* suggested window size */ - usize_t sprevsz; /* small string, previous window - size (power of 2) */ - usize_t sprevmask; /* small string, previous window - size mask */ - usize_t iopt_size; - usize_t iopt_unlimited; - usize_t srcwin_maxsz; - - /* general configuration */ - xd3_getblk_func *getblk; /* set nxtblk, nxtblkno to scanblkno */ - xd3_alloc_func *alloc; /* malloc function */ - xd3_free_func *free; /* free function */ - void* opaque; /* private data object passed to - alloc, free, and getblk */ - int flags; /* various options */ - - /* secondary compressor configuration */ - xd3_sec_cfg sec_data; /* Secondary compressor config: data */ - xd3_sec_cfg sec_inst; /* Secondary compressor config: inst */ - xd3_sec_cfg sec_addr; /* Secondary compressor config: addr */ - - xd3_smatcher smatcher; - - usize_t *large_table; /* table of large checksums */ - xd3_hash_cfg large_hash; /* large hash config */ - - usize_t *small_table; /* table of small checksums */ - xd3_slist *small_prev; /* table of previous offsets, - circular linked list */ - int small_reset; /* true if small table should - be reset */ - - xd3_hash_cfg small_hash; /* small hash config */ - xd3_addr_cache acache; /* the vcdiff address cache */ - xd3_encode_state enc_state; /* state of the encoder */ - - usize_t taroff; /* base offset of the target input */ - usize_t input_position; /* current input position */ - usize_t min_match; /* current minimum match - length, avoids redundent - matches */ - usize_t unencoded_offset; /* current input, first - * unencoded offset. this value - * is <= the first instruction's - * position in the iopt buffer, - * if there is at least one - * match in the buffer. */ - - // SRCWIN - // these variables plus srcwin_maxsz above (set by config) - int srcwin_decided; /* boolean: true if the - srclen,srcbase have been - decided. */ - xoff_t srcwin_cksum_pos; /* Source checksum position */ - - // MATCH - xd3_match_state match_state; /* encoder match state */ - xoff_t match_srcpos; /* current match source - position relative to - srcbase */ - xoff_t match_last_srcpos; /* previously attempted - * srcpos, to avoid loops. */ - xoff_t match_minaddr; /* smallest matching address to - * set window params (reset each - * window xd3_encode_reset) */ - xoff_t match_maxaddr; /* largest matching address to - * set window params (reset each - * window xd3_encode_reset) */ - usize_t match_back; /* match extends back so far */ - usize_t match_maxback; /* match extends back maximum */ - usize_t match_fwd; /* match extends forward so far */ - usize_t match_maxfwd; /* match extends forward maximum */ - - xoff_t maxsrcaddr; /* address of the last source - match (across windows) */ - - uint8_t *buf_in; /* for saving buffered input */ - usize_t buf_avail; /* amount of saved input */ - const uint8_t *buf_leftover; /* leftover content of next_in - (i.e., user's buffer) */ - usize_t buf_leftavail; /* amount of leftover content */ - - xd3_output *enc_current; /* current output buffer */ - xd3_output *enc_free; /* free output buffers */ - xd3_output *enc_heads[4]; /* array of encoded outputs: - head of chain */ - xd3_output *enc_tails[4]; /* array of encoded outputs: - tail of chain */ - uint32_t recode_adler32; /* set the adler32 checksum - * during "recode". */ - - xd3_rlist iopt_used; /* instruction optimizing buffer */ - xd3_rlist iopt_free; - xd3_rinst *iout; /* next single instruction */ - xd3_iopt_buflist *iopt_alloc; - - const uint8_t *enc_appheader; /* application header to encode */ - usize_t enc_appheadsz; /* application header size */ - - /* decoder stuff */ - xd3_decode_state dec_state; /* current DEC_XXX value */ - usize_t dec_hdr_ind; /* VCDIFF header indicator */ - usize_t dec_win_ind; /* VCDIFF window indicator */ - usize_t dec_del_ind; /* VCDIFF delta indicator */ - - uint8_t dec_magic[4]; /* First four bytes */ - usize_t dec_magicbytes; /* Magic position. */ - - usize_t dec_secondid; /* Optional secondary compressor ID. */ - - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_codetblsz; /* Optional code table: length. */ - uint8_t *dec_codetbl; /* Optional code table: storage. */ - usize_t dec_codetblbytes; /* Optional code table: position. */ - - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_appheadsz; /* Optional application header: - size. */ - uint8_t *dec_appheader; /* Optional application header: - storage */ - usize_t dec_appheadbytes; /* Optional application header: - position. */ - - usize_t dec_cksumbytes; /* Optional checksum: position. */ - uint8_t dec_cksum[4]; /* Optional checksum: storage. */ - uint32_t dec_adler32; /* Optional checksum: value. */ - - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_cpylen; /* length of copy window - (VCD_SOURCE or VCD_TARGET) */ - xoff_t dec_cpyoff; /* offset of copy window - (VCD_SOURCE or VCD_TARGET) */ - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_enclen; /* length of delta encoding */ - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_tgtlen; /* length of target window */ - -#if USE_UINT64 - uint64_t dec_64part; /* part of a decoded uint64_t */ -#endif -#if USE_UINT32 - uint32_t dec_32part; /* part of a decoded uint32_t */ -#endif - - xoff_t dec_winstart; /* offset of the start of - current target window */ - xoff_t dec_window_count; /* == current_window + 1 in - DEC_FINISH */ - usize_t dec_winbytes; /* bytes of the three sections - so far consumed */ - usize_t dec_hdrsize; /* VCDIFF + app header size */ - - const uint8_t *dec_tgtaddrbase; /* Base of decoded target - addresses (addr >= - dec_cpylen). */ - const uint8_t *dec_cpyaddrbase; /* Base of decoded copy - addresses (addr < - dec_cpylen). */ - - usize_t dec_position; /* current decoder position - counting the cpylen - offset */ - usize_t dec_maxpos; /* maximum decoder position - counting the cpylen - offset */ - xd3_hinst dec_current1; /* current instruction */ - xd3_hinst dec_current2; /* current instruction */ - - uint8_t *dec_buffer; /* Decode buffer */ - uint8_t *dec_lastwin; /* In case of VCD_TARGET, the - last target window. */ - usize_t dec_lastlen; /* length of the last target - window */ - xoff_t dec_laststart; /* offset of the start of last - target window */ - usize_t dec_lastspace; /* allocated space of last - target window, for reuse */ - - xd3_desect inst_sect; /* staging area for decoding - window sections */ - xd3_desect addr_sect; - xd3_desect data_sect; - - xd3_code_table_func *code_table_func; - xd3_comp_table_func *comp_table_func; - const xd3_dinst *code_table; - const xd3_code_table_desc *code_table_desc; - xd3_dinst *code_table_alloc; - - /* secondary compression */ - const xd3_sec_type *sec_type; - xd3_sec_stream *sec_stream_d; - xd3_sec_stream *sec_stream_i; - xd3_sec_stream *sec_stream_a; - - /* state for reconstructing whole files (e.g., for merge), this only - * supports loading USIZE_T_MAX instructions, adds, etc. */ - xd3_whole_state whole_target; - - /* statistics */ - xoff_t n_scpy; - xoff_t n_tcpy; - xoff_t n_add; - xoff_t n_run; - - xoff_t l_scpy; - xoff_t l_tcpy; - xoff_t l_add; - xoff_t l_run; - - usize_t i_slots_used; - -#if XD3_DEBUG - usize_t large_ckcnt; - - /* memory usage */ - usize_t alloc_cnt; - usize_t free_cnt; -#endif -}; - -/************************************************************************** - PUBLIC FUNCTIONS - **************************************************************************/ - -/* This function configures an xd3_stream using the provided in-memory - * input buffer, source buffer, output buffer, and flags. The output - * array must be large enough or else ENOSPC will be returned. This - * is the simplest in-memory encoding interface. */ -int xd3_encode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output_buffer, - usize_t *output_size, - usize_t avail_output, - int flags); - -/* The reverse of xd3_encode_memory. */ -int xd3_decode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output_buf, - usize_t *output_size, - usize_t avail_output, - int flags); - -/* This function encodes an in-memory input. Everything else about - * the xd3_stream is configurable. The output array must be large - * enough to hold the output or else ENOSPC is returned. The source - * (if any) should be set using xd3_set_source() with a single-block - * xd3_source. This calls the underlying non-blocking interface, - * xd3_encode_input(), handling the necessary input/output states. - * This method be considered a reference for any application using - * xd3_encode_input() directly. - * - * xd3_stream stream; - * xd3_config config; - * xd3_source src; - * - * memset (& src, 0, sizeof (src)); - * memset (& stream, 0, sizeof (stream)); - * memset (& config, 0, sizeof (config)); - * - * if (source != NULL) - * { - * src.size = source_size; - * src.blksize = source_size; - * src.curblkno = 0; - * src.onblk = source_size; - * src.curblk = source; - * xd3_set_source(&stream, &src); - * } - * - * config.flags = flags; - * config.srcwin_maxsz = source_size; - * config.winsize = input_size; - * - * ... set smatcher, appheader, encoding-table, compression-level, etc. - * - * xd3_config_stream(&stream, &config); - * xd3_encode_stream(&stream, ...); - * xd3_free_stream(&stream); - * - * DO NOT USE except for testing. These methods are allocate bad buffer sizes. - */ -int xd3_encode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t avail_output); - -/* The reverse of xd3_encode_stream. */ -int xd3_decode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t avail_size); - -/* This is the non-blocking interface. - * - * Handling input and output states is the same for encoding or - * decoding using the xd3_avail_input() and xd3_consume_output() - * routines, inlined below. - * - * Return values: - * - * XD3_INPUT: the process requires more input: call - * xd3_avail_input() then repeat - * - * XD3_OUTPUT: the process has more output: read stream->next_out, - * stream->avail_out, then call xd3_consume_output(), - * then repeat - * - * XD3_GOTHEADER: (decoder-only) notification returned following the - * VCDIFF header and first window header. the decoder - * may use the header to configure itself. - * - * XD3_WINSTART: a general notification returned once for each - * window except the 0-th window, which is implied by - * XD3_GOTHEADER. It is recommended to use a - * switch-stmt such as: - * - * ... - * again: - * switch ((ret = xd3_decode_input (stream))) { - * case XD3_GOTHEADER: { - * assert(stream->current_window == 0); - * stuff; - * } - * // fallthrough - * case XD3_WINSTART: { - * something(stream->current_window); - * goto again; - * } - * ... - * - * XD3_WINFINISH: a general notification, following the complete - * input & output of a window. at this point, - * stream->total_in and stream->total_out are consistent - * for either encoding or decoding. - * - * XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value - * is returned to initiate a non-blocking source read. - */ -int xd3_decode_input (xd3_stream *stream); -int xd3_encode_input (xd3_stream *stream); - -/* The xd3_config structure is used to initialize a stream - all data - * is copied into stream so config may be a temporary variable. See - * the [documentation] or comments on the xd3_config structure. */ -int xd3_config_stream (xd3_stream *stream, - xd3_config *config); - -/* Since Xdelta3 doesn't open any files, xd3_close_stream is just an - * error check that the stream is in a proper state to be closed: this - * means the encoder is flushed and the decoder is at a window - * boundary. The application is responsible for freeing any of the - * resources it supplied. */ -int xd3_close_stream (xd3_stream *stream); - -/* This arranges for closes the stream to succeed. Does not free the - * stream.*/ -void xd3_abort_stream (xd3_stream *stream); - -/* xd3_free_stream frees all memory allocated for the stream. The - * application is responsible for freeing any of the resources it - * supplied. */ -void xd3_free_stream (xd3_stream *stream); - -/* This function informs the encoder or decoder that source matching - * (i.e., delta-compression) is possible. For encoding, this should - * be called before the first xd3_encode_input. A NULL source is - * ignored. For decoding, this should be called before the first - * window is decoded, but the appheader may be read first - * (XD3_GOTHEADER). After decoding the header, call xd3_set_source() - * if you have a source file. Note: if (stream->dec_win_ind & VCD_SOURCE) - * is true, it means the first window expects there to be a source file. - */ -int xd3_set_source (xd3_stream *stream, - xd3_source *source); - -/* This should be called before the first call to xd3_encode_input() - * to include application-specific data in the VCDIFF header. */ -void xd3_set_appheader (xd3_stream *stream, - const uint8_t *data, - usize_t size); - -/* xd3_get_appheader may be called in the decoder after XD3_GOTHEADER. - * For convenience, the decoder always adds a single byte padding to - * the end of the application header, which is set to zero in case the - * application header is a string. */ -int xd3_get_appheader (xd3_stream *stream, - uint8_t **data, - usize_t *size); - -/* To generate a VCDIFF encoded delta with xd3_encode_init() from - * another format, use: - * - * xd3_encode_init_partial() -- initialze encoder state (w/o hash tables) - * xd3_init_cache() -- reset VCDIFF address cache - * xd3_found_match() -- to report a copy instruction - * - * set stream->enc_state to ENC_INSTR and call xd3_encode_input as usual. - */ -int xd3_encode_init_partial (xd3_stream *stream); -void xd3_init_cache (xd3_addr_cache* acache); -int xd3_found_match (xd3_stream *stream, - usize_t pos, usize_t size, - xoff_t addr, int is_source); - -/* Gives an error string for xdelta3-speficic errors, returns NULL for - system errors */ -const char* xd3_strerror (int ret); - -/* For convenience, zero & initialize the xd3_config structure with - specified flags. */ -static inline -void xd3_init_config (xd3_config *config, - int flags) -{ - memset (config, 0, sizeof (*config)); - config->flags = flags; -} - -/* This supplies some input to the stream. */ -static inline -void xd3_avail_input (xd3_stream *stream, - const uint8_t *idata, - usize_t isize) -{ - /* Even if isize is zero, the code expects a non-NULL idata. Why? - * It uses this value to determine whether xd3_avail_input has ever - * been called. If xd3_encode_input is called before - * xd3_avail_input it will return XD3_INPUT right away without - * allocating a stream->winsize buffer. This is to avoid an - * unwanted allocation. */ - XD3_ASSERT (idata != NULL || isize == 0); - - stream->next_in = idata; - stream->avail_in = isize; -} - -/* This acknowledges receipt of output data, must be called after any - * XD3_OUTPUT return. */ -static inline -void xd3_consume_output (xd3_stream *stream) -{ - stream->avail_out = 0; -} - -/* These are set for each XD3_WINFINISH return. */ -static inline -int xd3_encoder_used_source (xd3_stream *stream) { - return stream->src != NULL && stream->src->srclen > 0; -} -static inline -xoff_t xd3_encoder_srcbase (xd3_stream *stream) { - return stream->src->srcbase; -} -static inline -usize_t xd3_encoder_srclen (xd3_stream *stream) { - return stream->src->srclen; -} - -/* Checks for legal flag changes. */ -static inline -void xd3_set_flags (xd3_stream *stream, int flags) -{ - /* The bitwise difference should contain only XD3_FLUSH or - XD3_SKIP_WINDOW */ - XD3_ASSERT(((flags ^ stream->flags) & ~(XD3_FLUSH | XD3_SKIP_WINDOW)) == 0); - stream->flags = flags; -} - -/* Gives some extra information about the latest library error, if any - is known. */ -static inline -const char* xd3_errstring (xd3_stream *stream) -{ - return stream->msg ? stream->msg : ""; -} - - -/* 64-bit divisions are expensive. on a 32bit platform, these show in - * a profile as __udivdi3(). these are all the xoff_t divisions: */ -static inline -void xd3_blksize_div (const xoff_t offset, - const xd3_source *source, - xoff_t *blkno, - usize_t *blkoff) { - *blkno = source->maskby ? - (offset >> source->shiftby) : - (offset / source->blksize); - *blkoff = source->maskby ? - (offset & source->maskby) : - (offset - *blkno * source->blksize); -} - -/* This function tells the number of bytes expected to be set in - * source->onblk after a getblk request. This is for convenience of - * handling a partial last block. Note that this is a relatively - * expensive function for 64-bit binaries on platforms w/o native - * 64-bit integers, so source->onlastblk is set to this value. - * TODO: force source->blksize to a power of two? */ -static inline -usize_t xd3_bytes_on_srcblk (xd3_source *source, xoff_t blkno) -{ - xoff_t s_1_div; - usize_t s_1_rem; - XD3_ASSERT (blkno < source->blocks); - - if (blkno != source->blocks - 1) - { - return source->blksize; - } - xd3_blksize_div(source->size - 1, source, &s_1_div, &s_1_rem); - return s_1_rem + 1; -} - -static inline -usize_t xd3_bytes_on_srcblk_fast (xd3_source *source, xoff_t blkno) -{ - return (blkno == source->blocks - 1 ? - source->onlastblk : - source->blksize); -} - -#endif /* _XDELTA3_H_ */ diff --git a/xdelta3.py b/xdelta3.py deleted file mode 100644 index d077192..0000000 --- a/xdelta3.py +++ /dev/null @@ -1,71 +0,0 @@ -# This file was automatically generated by SWIG (http://www.swig.org). -# Version 1.3.31 -# -# Don't modify this file, modify the SWIG interface instead. -# This file is compatible with both classic and new-style classes. - -import _xdelta3 -import new -new_instancemethod = new.instancemethod -try: - _swig_property = property -except NameError: - pass # Python < 2.2 doesn't have 'property'. -def _swig_setattr_nondynamic(self,class_type,name,value,static=1): - if (name == "thisown"): return self.this.own(value) - if (name == "this"): - if type(value).__name__ == 'PySwigObject': - self.__dict__[name] = value - return - method = class_type.__swig_setmethods__.get(name,None) - if method: return method(self,value) - if (not static) or hasattr(self,name): - self.__dict__[name] = value - else: - raise AttributeError("You cannot add attributes to %s" % self) - -def _swig_setattr(self,class_type,name,value): - return _swig_setattr_nondynamic(self,class_type,name,value,0) - -def _swig_getattr(self,class_type,name): - if (name == "thisown"): return self.this.own() - method = class_type.__swig_getmethods__.get(name,None) - if method: return method(self) - raise AttributeError,name - -def _swig_repr(self): - try: strthis = "proxy of " + self.this.__repr__() - except: strthis = "" - return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) - -import types -try: - _object = types.ObjectType - _newclass = 1 -except AttributeError: - class _object : pass - _newclass = 0 -del types - - -xd3_encode_memory = _xdelta3.xd3_encode_memory -xd3_decode_memory = _xdelta3.xd3_decode_memory -xd3_main_cmdline = _xdelta3.xd3_main_cmdline -XD3_SEC_DJW = _xdelta3.XD3_SEC_DJW -XD3_SEC_FGK = _xdelta3.XD3_SEC_FGK -XD3_SEC_NODATA = _xdelta3.XD3_SEC_NODATA -XD3_SEC_NOINST = _xdelta3.XD3_SEC_NOINST -XD3_SEC_NOADDR = _xdelta3.XD3_SEC_NOADDR -XD3_ADLER32 = _xdelta3.XD3_ADLER32 -XD3_ADLER32_NOVER = _xdelta3.XD3_ADLER32_NOVER -XD3_ALT_CODE_TABLE = _xdelta3.XD3_ALT_CODE_TABLE -XD3_NOCOMPRESS = _xdelta3.XD3_NOCOMPRESS -XD3_BEGREEDY = _xdelta3.XD3_BEGREEDY -XD3_COMPLEVEL_SHIFT = _xdelta3.XD3_COMPLEVEL_SHIFT -XD3_COMPLEVEL_MASK = _xdelta3.XD3_COMPLEVEL_MASK -XD3_COMPLEVEL_1 = _xdelta3.XD3_COMPLEVEL_1 -XD3_COMPLEVEL_3 = _xdelta3.XD3_COMPLEVEL_3 -XD3_COMPLEVEL_6 = _xdelta3.XD3_COMPLEVEL_6 -XD3_COMPLEVEL_9 = _xdelta3.XD3_COMPLEVEL_9 - - diff --git a/xdelta3.swig b/xdelta3.swig deleted file mode 100644 index 2ef4306..0000000 --- a/xdelta3.swig +++ /dev/null @@ -1,93 +0,0 @@ -%module xdelta3 -%import cstring.i -%import argcargv.i -%{ -#include "xdelta3.h" - -int xd3_main_cmdline (int ARGC, char **ARGV); - -#undef SWIG_init -#undef SWIG_name - -#define SWIG_init initxdelta3 -#define SWIG_name "xdelta3" - -%} - -%cstring_input_binary(const char *input, unsigned int input_size); -%cstring_input_binary(const char *source, unsigned int source_size); - -%define %max_output_withsize(TYPEMAP, SIZE, MAXSIZE) -%typemap(in) MAXSIZE (unsigned int alloc_size) { - $1 = alloc_size = PyInt_AsLong(obj2); -} -%typemap(in,numinputs=0) (TYPEMAP, SIZE) { -} -%typemap(check) (TYPEMAP, SIZE) { - // alloc_size input is #7th position in xd3_xxcode_memory() - $1 = malloc(alloc_size7); - $2 = &alloc_size7; -} -%typemap(argout,fragment="t_output_helper") (TYPEMAP, SIZE) { - if (result == 0) { - PyObject *o; - // alloc_size7 now carries actual size - o = PyString_FromStringAndSize($1,alloc_size7); - $result = t_output_helper($result,o); - } else { - $result = t_output_helper($result,Py_None); - } - free($1); -} -%typemap(default) int flags { - $1 = 0; -} -%enddef - -%max_output_withsize(char *output_buf, unsigned int *output_size, unsigned int max_output); - -int xd3_encode_memory (const char *input, - unsigned int input_size, - const char *source, - unsigned int source_size, - char *output_buf, - unsigned int *output_size, - unsigned int max_output, - int flags); - -int xd3_decode_memory (const char *input, - unsigned int input_size, - const char *source, - unsigned int source_size, - char *output_buf, - unsigned int *output_size, - unsigned int max_output, - int flags); - -int xd3_main_cmdline (int ARGC, char **ARGV); - -/* Is this the right way? */ -enum { - /*XD3_JUST_HDR,*/ - /*XD3_SKIP_WINDOW,*/ - /*XD3_SKIP_EMIT,*/ - /*XD3_FLUSH,*/ - XD3_SEC_DJW, - XD3_SEC_FGK, - /*XD3_SEC_TYPE,*/ - XD3_SEC_NODATA, - XD3_SEC_NOINST, - XD3_SEC_NOADDR, - /*XD3_SEC_OTHER,*/ - XD3_ADLER32, - XD3_ADLER32_NOVER, - XD3_ALT_CODE_TABLE, - XD3_NOCOMPRESS, - XD3_BEGREEDY, - XD3_COMPLEVEL_SHIFT, - XD3_COMPLEVEL_MASK, - XD3_COMPLEVEL_1, - XD3_COMPLEVEL_3, - XD3_COMPLEVEL_6, - XD3_COMPLEVEL_9, -}; diff --git a/xdelta3.vcproj b/xdelta3.vcproj deleted file mode 100644 index 5dfb477..0000000 --- a/xdelta3.vcproj +++ /dev/null @@ -1,236 +0,0 @@ -<?xml version="1.0" encoding="Windows-1252"?> -<VisualStudioProject - ProjectType="Visual C++" - Version="8.00" - Name="xdelta3" - ProjectGUID="{7F30EDF1-4493-4E47-8664-0661516BC9E4}" - Keyword="Win32Proj" - > - <Platforms> - <Platform - Name="Win32" - /> - </Platforms> - <ToolFiles> - </ToolFiles> - <Configurations> - <Configuration - Name="Debug|Win32" - OutputDirectory="Debug" - IntermediateDirectory="Debug" - ConfigurationType="1" - > - <Tool - Name="VCPreBuildEventTool" - /> - <Tool - Name="VCCustomBuildTool" - /> - <Tool - Name="VCXMLDataGeneratorTool" - /> - <Tool - Name="VCWebServiceProxyGeneratorTool" - /> - <Tool - Name="VCMIDLTool" - /> - <Tool - Name="VCCLCompilerTool" - AdditionalOptions="/DXD3_DEBUG=0 /DXD3_USE_LARGEFILE64=1 /DREGRESSION_TEST=1 /DSECONDARY_DJW=1 /DSECONDARY_FGK=1 /DXD3_MAIN=1 /DXD3_WIN32=1 /DEXTERNAL_COMPRESSION=0 /DXD3_STDIO=0 /DXD3_POSIX=0 /D_CRT_SECURE_NO_DEPRECATE" - Optimization="0" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;" - MinimalRebuild="true" - BasicRuntimeChecks="3" - RuntimeLibrary="3" - UsePrecompiledHeader="0" - WarningLevel="3" - Detect64BitPortabilityProblems="true" - DebugInformationFormat="4" - /> - <Tool - Name="VCManagedResourceCompilerTool" - /> - <Tool - Name="VCResourceCompilerTool" - /> - <Tool - Name="VCPreLinkEventTool" - /> - <Tool - Name="VCLinkerTool" - LinkIncremental="2" - GenerateDebugInformation="true" - SubSystem="1" - TargetMachine="1" - /> - <Tool - Name="VCALinkTool" - /> - <Tool - Name="VCManifestTool" - /> - <Tool - Name="VCXDCMakeTool" - /> - <Tool - Name="VCBscMakeTool" - /> - <Tool - Name="VCFxCopTool" - /> - <Tool - Name="VCAppVerifierTool" - /> - <Tool - Name="VCWebDeploymentTool" - /> - <Tool - Name="VCPostBuildEventTool" - /> - </Configuration> - <Configuration - Name="Release|Win32" - OutputDirectory="Release" - IntermediateDirectory="Release" - ConfigurationType="1" - WholeProgramOptimization="1" - > - <Tool - Name="VCPreBuildEventTool" - /> - <Tool - Name="VCCustomBuildTool" - /> - <Tool - Name="VCXMLDataGeneratorTool" - /> - <Tool - Name="VCWebServiceProxyGeneratorTool" - /> - <Tool - Name="VCMIDLTool" - /> - <Tool - Name="VCCLCompilerTool" - AdditionalOptions="/DXD3_DEBUG=0 /DXD3_USE_LARGEFILE64=1 /DREGRESSION_TEST=1 /DSECONDARY_DJW=1 /DSECONDARY_FGK=1 /DXD3_MAIN=1 /DXD3_WIN32=1 /DEXTERNAL_COMPRESSION=0 /DXD3_STDIO=0 /DXD3_POSIX=0 /D_CRT_SECURE_NO_DEPRECATE" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE;" - RuntimeLibrary="0" - UsePrecompiledHeader="0" - WarningLevel="3" - Detect64BitPortabilityProblems="true" - DebugInformationFormat="3" - /> - <Tool - Name="VCManagedResourceCompilerTool" - /> - <Tool - Name="VCResourceCompilerTool" - /> - <Tool - Name="VCPreLinkEventTool" - /> - <Tool - Name="VCLinkerTool" - LinkIncremental="2" - GenerateDebugInformation="true" - SubSystem="1" - OptimizeReferences="2" - EnableCOMDATFolding="2" - TargetMachine="1" - /> - <Tool - Name="VCALinkTool" - /> - <Tool - Name="VCManifestTool" - /> - <Tool - Name="VCXDCMakeTool" - /> - <Tool - Name="VCBscMakeTool" - /> - <Tool - Name="VCFxCopTool" - /> - <Tool - Name="VCAppVerifierTool" - /> - <Tool - Name="VCWebDeploymentTool" - /> - <Tool - Name="VCPostBuildEventTool" - /> - </Configuration> - </Configurations> - <References> - </References> - <Files> - <Filter - Name="Header Files" - Filter="h;hpp;hxx;hm;inl;inc;xsd" - UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" - > - <File - RelativePath=".\xdelta3-cfgs.h" - > - </File> - <File - RelativePath=".\xdelta3-decode.h" - > - </File> - <File - RelativePath=".\xdelta3-djw.h" - > - </File> - <File - RelativePath=".\xdelta3-fgk.h" - > - </File> - <File - RelativePath=".\xdelta3-list.h" - > - </File> - <File - RelativePath=".\xdelta3-main.h" - > - </File> - <File - RelativePath=".\xdelta3-python.h" - > - </File> - <File - RelativePath=".\xdelta3-second.h" - > - </File> - <File - RelativePath=".\xdelta3-test.h" - > - </File> - <File - RelativePath=".\xdelta3.h" - > - </File> - </Filter> - <Filter - Name="Resource Files" - Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx" - UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}" - > - </Filter> - <Filter - Name="Source Files" - Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx" - UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" - > - <File - RelativePath=".\xdelta3.c" - > - </File> - </Filter> - </Files> - <Globals> - </Globals> -</VisualStudioProject> diff --git a/xdelta3.wxi b/xdelta3.wxi deleted file mode 100644 index 2ef8426..0000000 --- a/xdelta3.wxi +++ /dev/null @@ -1,7 +0,0 @@ -<Include> - <?define PRODUCT_ID=60131be5-be4d-4975-9108-dd0be735890d ?> - <?define PACKAGE_ID=82bf21ca-ee08-4701-ab78-37210dac82ce ?> - <?define COMPONENT_ID=85bc3206-05f8-41f8-b500-6ea32e5d6a8f ?> - <?define MANUAL_ID=07f387bc-a0c5-4af9-88db-1a84443f1fc5 ?> - <?define SOURCE_ID=4e1503a9-3ed1-4e06-b0c0-890462b1a4fd ?> -</Include> diff --git a/xdelta3.wxs b/xdelta3.wxs deleted file mode 100644 index 5e2d05c..0000000 --- a/xdelta3.wxs +++ /dev/null @@ -1,131 +0,0 @@ -<?xml version='1.0'?> -<?include $(sys.SOURCEFILEDIR)\xdelta3.wxi ?> - -<Wix xmlns='http://schemas.microsoft.com/wix/2003/01/wi'> - <Product Id='$(var.PRODUCT_ID)' - Name='Xdelta 3.0u' - Language='1033' - Codepage='1252' - Version='3.0.1.1' - Manufacturer='Josh.MacDonald@Gmail.Com'> - - <Package Id='$(var.PACKAGE_ID)' - Keywords='Installer' - Description='Xdelta 3.0u' - Comments='http://xdelta.org' - Manufacturer='Josh.MacDonald@Gmail.Com' - InstallerVersion='300' - Languages='1033' - Compressed='yes' /> - - <Media Id='1' - Cabinet='xdelta30t.cab' - EmbedCab='yes' /> - - <Directory Id='TARGETDIR' Name='SourceDir'> - <Directory Id='ProgramFilesFolder' Name='PFiles'> - <Directory Id='Xdelta' - Name='Xdelta'> - - <Component Id='Main' - Guid='$(var.COMPONENT_ID)'> - <File Id='XdeltaEXE' - Name='xdelt30t' - LongName='xdelta30t.exe' - DiskId='1' - Source='G:\jmacd\svn\xdelta3\Release\xdelta3.exe' - Vital='yes'> - </File> - </Component> - - <Component Id='Readme' - Guid='$(var.MANUAL_ID)'> - <File Id='Readme' - Name='readme.txt' - LongName='readme.txt' - DiskId='1' - Source='G:\jmacd\svn\xdelta3\readme.txt' - Vital='yes'> - <Shortcut Id="startupmenuReadme" - Directory="ProgramMenuDir" - Name="readme.txt" - LongName="Xdelta3 readme.txt" - /> - </File> - </Component> - - <Component Id='Copyright' - Guid='$(var.MANUAL_ID)'> - <File Id='Copyright' - Name='COPYING' - LongName='COPYING' - DiskId='1' - Source='G:\jmacd\svn\xdelta3\COPYING' - Vital='yes'> - <Shortcut Id="startupmenuCopyright" - Directory="ProgramMenuDir" - Name="COPYING" - LongName="GNU Public License" - /> - </File> - </Component> - - <Component Id='Source' - Guid='$(var.SOURCE_ID)'> - <File Id='Source' - Name='xdelt30t.zip' - LongName='xdelta3.0u.zip' - DiskId='1' - Source='G:\jmacd\svn\xdelta3\xdelta3.0u.zip' - Vital='yes'> - <Shortcut Id="startupmenuSource" - Directory="ProgramMenuDir" - Name="xdelt30t.zip" - LongName="xdelta3.0u.zip" - /> - </File> - </Component> - - </Directory> - </Directory> - - <Directory Id="ProgramMenuFolder" Name="PMenu" LongName="Programs"> - <Directory Id="ProgramMenuDir" - Name="xdelt30t" - LongName="Xdelta 3.0u"> - </Directory> - </Directory> - -<!-- <Merge Id='CRT' --> -<!-- Language='0' --> -<!-- DiskId='1' --> -<!-- src='C:\Program Files\Common Files\Merge Modules\microsoft_vc80_crt_x86.msm' --> -<!-- /> --> -<!-- <Merge Id='CRT Policy' --> -<!-- Language='0' --> -<!-- DiskId='1' --> -<!-- src='C:\Program Files\Common Files\Merge Modules\policy_8_0_Microsoft_VC80_CRT_x86.msm' --> -<!-- /> --> - </Directory> - - <Feature Id='Complete' - Level='1'> - <ComponentRef Id='Main' /> - <ComponentRef Id='Readme' /> - <ComponentRef Id='Copyright' /> - <ComponentRef Id='Source' /> - </Feature> - -<!-- <Feature Id='CRT_WinSXS' Title='CRT WinSXS' Level='1'> --> -<!-- <MergeRef Id='CRT' /> --> -<!-- <MergeRef Id='CRT Policy' /> --> -<!-- </Feature> --> - - <InstallExecuteSequence> - <RemoveRegistryValues/> - <RemoveFiles/> - <InstallFiles/> - <WriteRegistryValues/> - </InstallExecuteSequence> - </Product> -</Wix> diff --git a/xdelta3_wrap.c b/xdelta3_wrap.c deleted file mode 100644 index 1ed9b5d..0000000 --- a/xdelta3_wrap.c +++ /dev/null @@ -1,3615 +0,0 @@ -/* ---------------------------------------------------------------------------- - * This file was automatically generated by SWIG (http://www.swig.org). - * Version 1.3.31 - * - * This file is not intended to be easily readable and contains a number of - * coding conventions designed to improve portability and efficiency. Do not make - * changes to this file unless you know what you are doing--modify the SWIG - * interface file instead. - * ----------------------------------------------------------------------------- */ - -#define SWIGPYTHON -#define SWIG_PYTHON_DIRECTOR_NO_VTABLE -/* ----------------------------------------------------------------------------- - * This section contains generic SWIG labels for method/variable - * declarations/attributes, and other compiler dependent labels. - * ----------------------------------------------------------------------------- */ - -/* template workaround for compilers that cannot correctly implement the C++ standard */ -#ifndef SWIGTEMPLATEDISAMBIGUATOR -# if defined(__SUNPRO_CC) -# if (__SUNPRO_CC <= 0x560) -# define SWIGTEMPLATEDISAMBIGUATOR template -# else -# define SWIGTEMPLATEDISAMBIGUATOR -# endif -# else -# define SWIGTEMPLATEDISAMBIGUATOR -# endif -#endif - -/* inline attribute */ -#ifndef SWIGINLINE -# if defined(__cplusplus) || (defined(__GNUC__) && !defined(__STRICT_ANSI__)) -# define SWIGINLINE inline -# else -# define SWIGINLINE -# endif -#endif - -/* attribute recognised by some compilers to avoid 'unused' warnings */ -#ifndef SWIGUNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define SWIGUNUSED __attribute__ ((__unused__)) -# else -# define SWIGUNUSED -# endif -# elif defined(__ICC) -# define SWIGUNUSED __attribute__ ((__unused__)) -# else -# define SWIGUNUSED -# endif -#endif - -#ifndef SWIGUNUSEDPARM -# ifdef __cplusplus -# define SWIGUNUSEDPARM(p) -# else -# define SWIGUNUSEDPARM(p) p SWIGUNUSED -# endif -#endif - -/* internal SWIG method */ -#ifndef SWIGINTERN -# define SWIGINTERN static SWIGUNUSED -#endif - -/* internal inline SWIG method */ -#ifndef SWIGINTERNINLINE -# define SWIGINTERNINLINE SWIGINTERN SWIGINLINE -#endif - -/* exporting methods */ -#if (__GNUC__ >= 4) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) -# ifndef GCC_HASCLASSVISIBILITY -# define GCC_HASCLASSVISIBILITY -# endif -#endif - -#ifndef SWIGEXPORT -# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) -# if defined(STATIC_LINKED) -# define SWIGEXPORT -# else -# define SWIGEXPORT __declspec(dllexport) -# endif -# else -# if defined(__GNUC__) && defined(GCC_HASCLASSVISIBILITY) -# define SWIGEXPORT __attribute__ ((visibility("default"))) -# else -# define SWIGEXPORT -# endif -# endif -#endif - -/* calling conventions for Windows */ -#ifndef SWIGSTDCALL -# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) -# define SWIGSTDCALL __stdcall -# else -# define SWIGSTDCALL -# endif -#endif - -/* Deal with Microsoft's attempt at deprecating C standard runtime functions */ -#if !defined(SWIG_NO_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) && !defined(_CRT_SECURE_NO_DEPRECATE) -# define _CRT_SECURE_NO_DEPRECATE -#endif - - -/* Python.h has to appear first */ -#include <Python.h> - -/* ----------------------------------------------------------------------------- - * swigrun.swg - * - * This file contains generic CAPI SWIG runtime support for pointer - * type checking. - * ----------------------------------------------------------------------------- */ - -/* This should only be incremented when either the layout of swig_type_info changes, - or for whatever reason, the runtime changes incompatibly */ -#define SWIG_RUNTIME_VERSION "3" - -/* define SWIG_TYPE_TABLE_NAME as "SWIG_TYPE_TABLE" */ -#ifdef SWIG_TYPE_TABLE -# define SWIG_QUOTE_STRING(x) #x -# define SWIG_EXPAND_AND_QUOTE_STRING(x) SWIG_QUOTE_STRING(x) -# define SWIG_TYPE_TABLE_NAME SWIG_EXPAND_AND_QUOTE_STRING(SWIG_TYPE_TABLE) -#else -# define SWIG_TYPE_TABLE_NAME -#endif - -/* - You can use the SWIGRUNTIME and SWIGRUNTIMEINLINE macros for - creating a static or dynamic library from the swig runtime code. - In 99.9% of the cases, swig just needs to declare them as 'static'. - - But only do this if is strictly necessary, ie, if you have problems - with your compiler or so. -*/ - -#ifndef SWIGRUNTIME -# define SWIGRUNTIME SWIGINTERN -#endif - -#ifndef SWIGRUNTIMEINLINE -# define SWIGRUNTIMEINLINE SWIGRUNTIME SWIGINLINE -#endif - -/* Generic buffer size */ -#ifndef SWIG_BUFFER_SIZE -# define SWIG_BUFFER_SIZE 1024 -#endif - -/* Flags for pointer conversions */ -#define SWIG_POINTER_DISOWN 0x1 - -/* Flags for new pointer objects */ -#define SWIG_POINTER_OWN 0x1 - - -/* - Flags/methods for returning states. - - The swig conversion methods, as ConvertPtr, return and integer - that tells if the conversion was successful or not. And if not, - an error code can be returned (see swigerrors.swg for the codes). - - Use the following macros/flags to set or process the returning - states. - - In old swig versions, you usually write code as: - - if (SWIG_ConvertPtr(obj,vptr,ty.flags) != -1) { - // success code - } else { - //fail code - } - - Now you can be more explicit as: - - int res = SWIG_ConvertPtr(obj,vptr,ty.flags); - if (SWIG_IsOK(res)) { - // success code - } else { - // fail code - } - - that seems to be the same, but now you can also do - - Type *ptr; - int res = SWIG_ConvertPtr(obj,(void **)(&ptr),ty.flags); - if (SWIG_IsOK(res)) { - // success code - if (SWIG_IsNewObj(res) { - ... - delete *ptr; - } else { - ... - } - } else { - // fail code - } - - I.e., now SWIG_ConvertPtr can return new objects and you can - identify the case and take care of the deallocation. Of course that - requires also to SWIG_ConvertPtr to return new result values, as - - int SWIG_ConvertPtr(obj, ptr,...) { - if (<obj is ok>) { - if (<need new object>) { - *ptr = <ptr to new allocated object>; - return SWIG_NEWOBJ; - } else { - *ptr = <ptr to old object>; - return SWIG_OLDOBJ; - } - } else { - return SWIG_BADOBJ; - } - } - - Of course, returning the plain '0(success)/-1(fail)' still works, but you can be - more explicit by returning SWIG_BADOBJ, SWIG_ERROR or any of the - swig errors code. - - Finally, if the SWIG_CASTRANK_MODE is enabled, the result code - allows to return the 'cast rank', for example, if you have this - - int food(double) - int fooi(int); - - and you call - - food(1) // cast rank '1' (1 -> 1.0) - fooi(1) // cast rank '0' - - just use the SWIG_AddCast()/SWIG_CheckState() - - - */ -#define SWIG_OK (0) -#define SWIG_ERROR (-1) -#define SWIG_IsOK(r) (r >= 0) -#define SWIG_ArgError(r) ((r != SWIG_ERROR) ? r : SWIG_TypeError) - -/* The CastRankLimit says how many bits are used for the cast rank */ -#define SWIG_CASTRANKLIMIT (1 << 8) -/* The NewMask denotes the object was created (using new/malloc) */ -#define SWIG_NEWOBJMASK (SWIG_CASTRANKLIMIT << 1) -/* The TmpMask is for in/out typemaps that use temporal objects */ -#define SWIG_TMPOBJMASK (SWIG_NEWOBJMASK << 1) -/* Simple returning values */ -#define SWIG_BADOBJ (SWIG_ERROR) -#define SWIG_OLDOBJ (SWIG_OK) -#define SWIG_NEWOBJ (SWIG_OK | SWIG_NEWOBJMASK) -#define SWIG_TMPOBJ (SWIG_OK | SWIG_TMPOBJMASK) -/* Check, add and del mask methods */ -#define SWIG_AddNewMask(r) (SWIG_IsOK(r) ? (r | SWIG_NEWOBJMASK) : r) -#define SWIG_DelNewMask(r) (SWIG_IsOK(r) ? (r & ~SWIG_NEWOBJMASK) : r) -#define SWIG_IsNewObj(r) (SWIG_IsOK(r) && (r & SWIG_NEWOBJMASK)) -#define SWIG_AddTmpMask(r) (SWIG_IsOK(r) ? (r | SWIG_TMPOBJMASK) : r) -#define SWIG_DelTmpMask(r) (SWIG_IsOK(r) ? (r & ~SWIG_TMPOBJMASK) : r) -#define SWIG_IsTmpObj(r) (SWIG_IsOK(r) && (r & SWIG_TMPOBJMASK)) - - -/* Cast-Rank Mode */ -#if defined(SWIG_CASTRANK_MODE) -# ifndef SWIG_TypeRank -# define SWIG_TypeRank unsigned long -# endif -# ifndef SWIG_MAXCASTRANK /* Default cast allowed */ -# define SWIG_MAXCASTRANK (2) -# endif -# define SWIG_CASTRANKMASK ((SWIG_CASTRANKLIMIT) -1) -# define SWIG_CastRank(r) (r & SWIG_CASTRANKMASK) -SWIGINTERNINLINE int SWIG_AddCast(int r) { - return SWIG_IsOK(r) ? ((SWIG_CastRank(r) < SWIG_MAXCASTRANK) ? (r + 1) : SWIG_ERROR) : r; -} -SWIGINTERNINLINE int SWIG_CheckState(int r) { - return SWIG_IsOK(r) ? SWIG_CastRank(r) + 1 : 0; -} -#else /* no cast-rank mode */ -# define SWIG_AddCast -# define SWIG_CheckState(r) (SWIG_IsOK(r) ? 1 : 0) -#endif - - - - -#include <string.h> - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void *(*swig_converter_func)(void *); -typedef struct swig_type_info *(*swig_dycast_func)(void **); - -/* Structure to store inforomation on one type */ -typedef struct swig_type_info { - const char *name; /* mangled name of this type */ - const char *str; /* human readable name of this type */ - swig_dycast_func dcast; /* dynamic cast function down a hierarchy */ - struct swig_cast_info *cast; /* linked list of types that can cast into this type */ - void *clientdata; /* language specific type data */ - int owndata; /* flag if the structure owns the clientdata */ -} swig_type_info; - -/* Structure to store a type and conversion function used for casting */ -typedef struct swig_cast_info { - swig_type_info *type; /* pointer to type that is equivalent to this type */ - swig_converter_func converter; /* function to cast the void pointers */ - struct swig_cast_info *next; /* pointer to next cast in linked list */ - struct swig_cast_info *prev; /* pointer to the previous cast */ -} swig_cast_info; - -/* Structure used to store module information - * Each module generates one structure like this, and the runtime collects - * all of these structures and stores them in a circularly linked list.*/ -typedef struct swig_module_info { - swig_type_info **types; /* Array of pointers to swig_type_info structures that are in this module */ - size_t size; /* Number of types in this module */ - struct swig_module_info *next; /* Pointer to next element in circularly linked list */ - swig_type_info **type_initial; /* Array of initially generated type structures */ - swig_cast_info **cast_initial; /* Array of initially generated casting structures */ - void *clientdata; /* Language specific module data */ -} swig_module_info; - -/* - Compare two type names skipping the space characters, therefore - "char*" == "char *" and "Class<int>" == "Class<int >", etc. - - Return 0 when the two name types are equivalent, as in - strncmp, but skipping ' '. -*/ -SWIGRUNTIME int -SWIG_TypeNameComp(const char *f1, const char *l1, - const char *f2, const char *l2) { - for (;(f1 != l1) && (f2 != l2); ++f1, ++f2) { - while ((*f1 == ' ') && (f1 != l1)) ++f1; - while ((*f2 == ' ') && (f2 != l2)) ++f2; - if (*f1 != *f2) return (*f1 > *f2) ? 1 : -1; - } - return (l1 - f1) - (l2 - f2); -} - -/* - Check type equivalence in a name list like <name1>|<name2>|... - Return 0 if not equal, 1 if equal -*/ -SWIGRUNTIME int -SWIG_TypeEquiv(const char *nb, const char *tb) { - int equiv = 0; - const char* te = tb + strlen(tb); - const char* ne = nb; - while (!equiv && *ne) { - for (nb = ne; *ne; ++ne) { - if (*ne == '|') break; - } - equiv = (SWIG_TypeNameComp(nb, ne, tb, te) == 0) ? 1 : 0; - if (*ne) ++ne; - } - return equiv; -} - -/* - Check type equivalence in a name list like <name1>|<name2>|... - Return 0 if equal, -1 if nb < tb, 1 if nb > tb -*/ -SWIGRUNTIME int -SWIG_TypeCompare(const char *nb, const char *tb) { - int equiv = 0; - const char* te = tb + strlen(tb); - const char* ne = nb; - while (!equiv && *ne) { - for (nb = ne; *ne; ++ne) { - if (*ne == '|') break; - } - equiv = (SWIG_TypeNameComp(nb, ne, tb, te) == 0) ? 1 : 0; - if (*ne) ++ne; - } - return equiv; -} - - -/* think of this as a c++ template<> or a scheme macro */ -#define SWIG_TypeCheck_Template(comparison, ty) \ - if (ty) { \ - swig_cast_info *iter = ty->cast; \ - while (iter) { \ - if (comparison) { \ - if (iter == ty->cast) return iter; \ - /* Move iter to the top of the linked list */ \ - iter->prev->next = iter->next; \ - if (iter->next) \ - iter->next->prev = iter->prev; \ - iter->next = ty->cast; \ - iter->prev = 0; \ - if (ty->cast) ty->cast->prev = iter; \ - ty->cast = iter; \ - return iter; \ - } \ - iter = iter->next; \ - } \ - } \ - return 0 - -/* - Check the typename -*/ -SWIGRUNTIME swig_cast_info * -SWIG_TypeCheck(const char *c, swig_type_info *ty) { - SWIG_TypeCheck_Template(strcmp(iter->type->name, c) == 0, ty); -} - -/* Same as previous function, except strcmp is replaced with a pointer comparison */ -SWIGRUNTIME swig_cast_info * -SWIG_TypeCheckStruct(swig_type_info *from, swig_type_info *into) { - SWIG_TypeCheck_Template(iter->type == from, into); -} - -/* - Cast a pointer up an inheritance hierarchy -*/ -SWIGRUNTIMEINLINE void * -SWIG_TypeCast(swig_cast_info *ty, void *ptr) { - return ((!ty) || (!ty->converter)) ? ptr : (*ty->converter)(ptr); -} - -/* - Dynamic pointer casting. Down an inheritance hierarchy -*/ -SWIGRUNTIME swig_type_info * -SWIG_TypeDynamicCast(swig_type_info *ty, void **ptr) { - swig_type_info *lastty = ty; - if (!ty || !ty->dcast) return ty; - while (ty && (ty->dcast)) { - ty = (*ty->dcast)(ptr); - if (ty) lastty = ty; - } - return lastty; -} - -/* - Return the name associated with this type -*/ -SWIGRUNTIMEINLINE const char * -SWIG_TypeName(const swig_type_info *ty) { - return ty->name; -} - -/* - Return the pretty name associated with this type, - that is an unmangled type name in a form presentable to the user. -*/ -SWIGRUNTIME const char * -SWIG_TypePrettyName(const swig_type_info *type) { - /* The "str" field contains the equivalent pretty names of the - type, separated by vertical-bar characters. We choose - to print the last name, as it is often (?) the most - specific. */ - if (!type) return NULL; - if (type->str != NULL) { - const char *last_name = type->str; - const char *s; - for (s = type->str; *s; s++) - if (*s == '|') last_name = s+1; - return last_name; - } - else - return type->name; -} - -/* - Set the clientdata field for a type -*/ -SWIGRUNTIME void -SWIG_TypeClientData(swig_type_info *ti, void *clientdata) { - swig_cast_info *cast = ti->cast; - /* if (ti->clientdata == clientdata) return; */ - ti->clientdata = clientdata; - - while (cast) { - if (!cast->converter) { - swig_type_info *tc = cast->type; - if (!tc->clientdata) { - SWIG_TypeClientData(tc, clientdata); - } - } - cast = cast->next; - } -} -SWIGRUNTIME void -SWIG_TypeNewClientData(swig_type_info *ti, void *clientdata) { - SWIG_TypeClientData(ti, clientdata); - ti->owndata = 1; -} - -/* - Search for a swig_type_info structure only by mangled name - Search is a O(log #types) - - We start searching at module start, and finish searching when start == end. - Note: if start == end at the beginning of the function, we go all the way around - the circular list. -*/ -SWIGRUNTIME swig_type_info * -SWIG_MangledTypeQueryModule(swig_module_info *start, - swig_module_info *end, - const char *name) { - swig_module_info *iter = start; - do { - if (iter->size) { - register size_t l = 0; - register size_t r = iter->size - 1; - do { - /* since l+r >= 0, we can (>> 1) instead (/ 2) */ - register size_t i = (l + r) >> 1; - const char *iname = iter->types[i]->name; - if (iname) { - register int compare = strcmp(name, iname); - if (compare == 0) { - return iter->types[i]; - } else if (compare < 0) { - if (i) { - r = i - 1; - } else { - break; - } - } else if (compare > 0) { - l = i + 1; - } - } else { - break; /* should never happen */ - } - } while (l <= r); - } - iter = iter->next; - } while (iter != end); - return 0; -} - -/* - Search for a swig_type_info structure for either a mangled name or a human readable name. - It first searches the mangled names of the types, which is a O(log #types) - If a type is not found it then searches the human readable names, which is O(#types). - - We start searching at module start, and finish searching when start == end. - Note: if start == end at the beginning of the function, we go all the way around - the circular list. -*/ -SWIGRUNTIME swig_type_info * -SWIG_TypeQueryModule(swig_module_info *start, - swig_module_info *end, - const char *name) { - /* STEP 1: Search the name field using binary search */ - swig_type_info *ret = SWIG_MangledTypeQueryModule(start, end, name); - if (ret) { - return ret; - } else { - /* STEP 2: If the type hasn't been found, do a complete search - of the str field (the human readable name) */ - swig_module_info *iter = start; - do { - register size_t i = 0; - for (; i < iter->size; ++i) { - if (iter->types[i]->str && (SWIG_TypeEquiv(iter->types[i]->str, name))) - return iter->types[i]; - } - iter = iter->next; - } while (iter != end); - } - - /* neither found a match */ - return 0; -} - -/* - Pack binary data into a string -*/ -SWIGRUNTIME char * -SWIG_PackData(char *c, void *ptr, size_t sz) { - static const char hex[17] = "0123456789abcdef"; - register const unsigned char *u = (unsigned char *) ptr; - register const unsigned char *eu = u + sz; - for (; u != eu; ++u) { - register unsigned char uu = *u; - *(c++) = hex[(uu & 0xf0) >> 4]; - *(c++) = hex[uu & 0xf]; - } - return c; -} - -/* - Unpack binary data from a string -*/ -SWIGRUNTIME const char * -SWIG_UnpackData(const char *c, void *ptr, size_t sz) { - register unsigned char *u = (unsigned char *) ptr; - register const unsigned char *eu = u + sz; - for (; u != eu; ++u) { - register char d = *(c++); - register unsigned char uu; - if ((d >= '0') && (d <= '9')) - uu = ((d - '0') << 4); - else if ((d >= 'a') && (d <= 'f')) - uu = ((d - ('a'-10)) << 4); - else - return (char *) 0; - d = *(c++); - if ((d >= '0') && (d <= '9')) - uu |= (d - '0'); - else if ((d >= 'a') && (d <= 'f')) - uu |= (d - ('a'-10)); - else - return (char *) 0; - *u = uu; - } - return c; -} - -/* - Pack 'void *' into a string buffer. -*/ -SWIGRUNTIME char * -SWIG_PackVoidPtr(char *buff, void *ptr, const char *name, size_t bsz) { - char *r = buff; - if ((2*sizeof(void *) + 2) > bsz) return 0; - *(r++) = '_'; - r = SWIG_PackData(r,&ptr,sizeof(void *)); - if (strlen(name) + 1 > (bsz - (r - buff))) return 0; - strcpy(r,name); - return buff; -} - -SWIGRUNTIME const char * -SWIG_UnpackVoidPtr(const char *c, void **ptr, const char *name) { - if (*c != '_') { - if (strcmp(c,"NULL") == 0) { - *ptr = (void *) 0; - return name; - } else { - return 0; - } - } - return SWIG_UnpackData(++c,ptr,sizeof(void *)); -} - -SWIGRUNTIME char * -SWIG_PackDataName(char *buff, void *ptr, size_t sz, const char *name, size_t bsz) { - char *r = buff; - size_t lname = (name ? strlen(name) : 0); - if ((2*sz + 2 + lname) > bsz) return 0; - *(r++) = '_'; - r = SWIG_PackData(r,ptr,sz); - if (lname) { - strncpy(r,name,lname+1); - } else { - *r = 0; - } - return buff; -} - -SWIGRUNTIME const char * -SWIG_UnpackDataName(const char *c, void *ptr, size_t sz, const char *name) { - if (*c != '_') { - if (strcmp(c,"NULL") == 0) { - memset(ptr,0,sz); - return name; - } else { - return 0; - } - } - return SWIG_UnpackData(++c,ptr,sz); -} - -#ifdef __cplusplus -} -#endif - -/* Errors in SWIG */ -#define SWIG_UnknownError -1 -#define SWIG_IOError -2 -#define SWIG_RuntimeError -3 -#define SWIG_IndexError -4 -#define SWIG_TypeError -5 -#define SWIG_DivisionByZero -6 -#define SWIG_OverflowError -7 -#define SWIG_SyntaxError -8 -#define SWIG_ValueError -9 -#define SWIG_SystemError -10 -#define SWIG_AttributeError -11 -#define SWIG_MemoryError -12 -#define SWIG_NullReferenceError -13 - - - - -/* Add PyOS_snprintf for old Pythons */ -#if PY_VERSION_HEX < 0x02020000 -# if defined(_MSC_VER) || defined(__BORLANDC__) || defined(_WATCOM) -# define PyOS_snprintf _snprintf -# else -# define PyOS_snprintf snprintf -# endif -#endif - -/* A crude PyString_FromFormat implementation for old Pythons */ -#if PY_VERSION_HEX < 0x02020000 - -#ifndef SWIG_PYBUFFER_SIZE -# define SWIG_PYBUFFER_SIZE 1024 -#endif - -static PyObject * -PyString_FromFormat(const char *fmt, ...) { - va_list ap; - char buf[SWIG_PYBUFFER_SIZE * 2]; - int res; - va_start(ap, fmt); - res = vsnprintf(buf, sizeof(buf), fmt, ap); - va_end(ap); - return (res < 0 || res >= (int)sizeof(buf)) ? 0 : PyString_FromString(buf); -} -#endif - -/* Add PyObject_Del for old Pythons */ -#if PY_VERSION_HEX < 0x01060000 -# define PyObject_Del(op) PyMem_DEL((op)) -#endif -#ifndef PyObject_DEL -# define PyObject_DEL PyObject_Del -#endif - -/* A crude PyExc_StopIteration exception for old Pythons */ -#if PY_VERSION_HEX < 0x02020000 -# ifndef PyExc_StopIteration -# define PyExc_StopIteration PyExc_RuntimeError -# endif -# ifndef PyObject_GenericGetAttr -# define PyObject_GenericGetAttr 0 -# endif -#endif -/* Py_NotImplemented is defined in 2.1 and up. */ -#if PY_VERSION_HEX < 0x02010000 -# ifndef Py_NotImplemented -# define Py_NotImplemented PyExc_RuntimeError -# endif -#endif - - -/* A crude PyString_AsStringAndSize implementation for old Pythons */ -#if PY_VERSION_HEX < 0x02010000 -# ifndef PyString_AsStringAndSize -# define PyString_AsStringAndSize(obj, s, len) {*s = PyString_AsString(obj); *len = *s ? strlen(*s) : 0;} -# endif -#endif - -/* PySequence_Size for old Pythons */ -#if PY_VERSION_HEX < 0x02000000 -# ifndef PySequence_Size -# define PySequence_Size PySequence_Length -# endif -#endif - - -/* PyBool_FromLong for old Pythons */ -#if PY_VERSION_HEX < 0x02030000 -static -PyObject *PyBool_FromLong(long ok) -{ - PyObject *result = ok ? Py_True : Py_False; - Py_INCREF(result); - return result; -} -#endif - -/* Py_ssize_t for old Pythons */ -/* This code is as recommended by: */ -/* http://www.python.org/dev/peps/pep-0353/#conversion-guidelines */ -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -# define PY_SSIZE_T_MAX INT_MAX -# define PY_SSIZE_T_MIN INT_MIN -#endif - -/* ----------------------------------------------------------------------------- - * error manipulation - * ----------------------------------------------------------------------------- */ - -SWIGRUNTIME PyObject* -SWIG_Python_ErrorType(int code) { - PyObject* type = 0; - switch(code) { - case SWIG_MemoryError: - type = PyExc_MemoryError; - break; - case SWIG_IOError: - type = PyExc_IOError; - break; - case SWIG_RuntimeError: - type = PyExc_RuntimeError; - break; - case SWIG_IndexError: - type = PyExc_IndexError; - break; - case SWIG_TypeError: - type = PyExc_TypeError; - break; - case SWIG_DivisionByZero: - type = PyExc_ZeroDivisionError; - break; - case SWIG_OverflowError: - type = PyExc_OverflowError; - break; - case SWIG_SyntaxError: - type = PyExc_SyntaxError; - break; - case SWIG_ValueError: - type = PyExc_ValueError; - break; - case SWIG_SystemError: - type = PyExc_SystemError; - break; - case SWIG_AttributeError: - type = PyExc_AttributeError; - break; - default: - type = PyExc_RuntimeError; - } - return type; -} - - -SWIGRUNTIME void -SWIG_Python_AddErrorMsg(const char* mesg) -{ - PyObject *type = 0; - PyObject *value = 0; - PyObject *traceback = 0; - - if (PyErr_Occurred()) PyErr_Fetch(&type, &value, &traceback); - if (value) { - PyObject *old_str = PyObject_Str(value); - PyErr_Clear(); - Py_XINCREF(type); - PyErr_Format(type, "%s %s", PyString_AsString(old_str), mesg); - Py_DECREF(old_str); - Py_DECREF(value); - } else { - PyErr_Format(PyExc_RuntimeError, mesg); - } -} - - - -#if defined(SWIG_PYTHON_NO_THREADS) -# if defined(SWIG_PYTHON_THREADS) -# undef SWIG_PYTHON_THREADS -# endif -#endif -#if defined(SWIG_PYTHON_THREADS) /* Threading support is enabled */ -# if !defined(SWIG_PYTHON_USE_GIL) && !defined(SWIG_PYTHON_NO_USE_GIL) -# if (PY_VERSION_HEX >= 0x02030000) /* For 2.3 or later, use the PyGILState calls */ -# define SWIG_PYTHON_USE_GIL -# endif -# endif -# if defined(SWIG_PYTHON_USE_GIL) /* Use PyGILState threads calls */ -# ifndef SWIG_PYTHON_INITIALIZE_THREADS -# define SWIG_PYTHON_INITIALIZE_THREADS PyEval_InitThreads() -# endif -# ifdef __cplusplus /* C++ code */ - class SWIG_Python_Thread_Block { - bool status; - PyGILState_STATE state; - public: - void end() { if (status) { PyGILState_Release(state); status = false;} } - SWIG_Python_Thread_Block() : status(true), state(PyGILState_Ensure()) {} - ~SWIG_Python_Thread_Block() { end(); } - }; - class SWIG_Python_Thread_Allow { - bool status; - PyThreadState *save; - public: - void end() { if (status) { PyEval_RestoreThread(save); status = false; }} - SWIG_Python_Thread_Allow() : status(true), save(PyEval_SaveThread()) {} - ~SWIG_Python_Thread_Allow() { end(); } - }; -# define SWIG_PYTHON_THREAD_BEGIN_BLOCK SWIG_Python_Thread_Block _swig_thread_block -# define SWIG_PYTHON_THREAD_END_BLOCK _swig_thread_block.end() -# define SWIG_PYTHON_THREAD_BEGIN_ALLOW SWIG_Python_Thread_Allow _swig_thread_allow -# define SWIG_PYTHON_THREAD_END_ALLOW _swig_thread_allow.end() -# else /* C code */ -# define SWIG_PYTHON_THREAD_BEGIN_BLOCK PyGILState_STATE _swig_thread_block = PyGILState_Ensure() -# define SWIG_PYTHON_THREAD_END_BLOCK PyGILState_Release(_swig_thread_block) -# define SWIG_PYTHON_THREAD_BEGIN_ALLOW PyThreadState *_swig_thread_allow = PyEval_SaveThread() -# define SWIG_PYTHON_THREAD_END_ALLOW PyEval_RestoreThread(_swig_thread_allow) -# endif -# else /* Old thread way, not implemented, user must provide it */ -# if !defined(SWIG_PYTHON_INITIALIZE_THREADS) -# define SWIG_PYTHON_INITIALIZE_THREADS -# endif -# if !defined(SWIG_PYTHON_THREAD_BEGIN_BLOCK) -# define SWIG_PYTHON_THREAD_BEGIN_BLOCK -# endif -# if !defined(SWIG_PYTHON_THREAD_END_BLOCK) -# define SWIG_PYTHON_THREAD_END_BLOCK -# endif -# if !defined(SWIG_PYTHON_THREAD_BEGIN_ALLOW) -# define SWIG_PYTHON_THREAD_BEGIN_ALLOW -# endif -# if !defined(SWIG_PYTHON_THREAD_END_ALLOW) -# define SWIG_PYTHON_THREAD_END_ALLOW -# endif -# endif -#else /* No thread support */ -# define SWIG_PYTHON_INITIALIZE_THREADS -# define SWIG_PYTHON_THREAD_BEGIN_BLOCK -# define SWIG_PYTHON_THREAD_END_BLOCK -# define SWIG_PYTHON_THREAD_BEGIN_ALLOW -# define SWIG_PYTHON_THREAD_END_ALLOW -#endif - -/* ----------------------------------------------------------------------------- - * Python API portion that goes into the runtime - * ----------------------------------------------------------------------------- */ - -#ifdef __cplusplus -extern "C" { -#if 0 -} /* cc-mode */ -#endif -#endif - -/* ----------------------------------------------------------------------------- - * Constant declarations - * ----------------------------------------------------------------------------- */ - -/* Constant Types */ -#define SWIG_PY_POINTER 4 -#define SWIG_PY_BINARY 5 - -/* Constant information structure */ -typedef struct swig_const_info { - int type; - char *name; - long lvalue; - double dvalue; - void *pvalue; - swig_type_info **ptype; -} swig_const_info; - -#ifdef __cplusplus -#if 0 -{ /* cc-mode */ -#endif -} -#endif - - -/* ----------------------------------------------------------------------------- - * See the LICENSE file for information on copyright, usage and redistribution - * of SWIG, and the README file for authors - http://www.swig.org/release.html. - * - * pyrun.swg - * - * This file contains the runtime support for Python modules - * and includes code for managing global variables and pointer - * type checking. - * - * ----------------------------------------------------------------------------- */ - -/* Common SWIG API */ - -/* for raw pointers */ -#define SWIG_Python_ConvertPtr(obj, pptr, type, flags) SWIG_Python_ConvertPtrAndOwn(obj, pptr, type, flags, 0) -#define SWIG_ConvertPtr(obj, pptr, type, flags) SWIG_Python_ConvertPtr(obj, pptr, type, flags) -#define SWIG_ConvertPtrAndOwn(obj,pptr,type,flags,own) SWIG_Python_ConvertPtrAndOwn(obj, pptr, type, flags, own) -#define SWIG_NewPointerObj(ptr, type, flags) SWIG_Python_NewPointerObj(ptr, type, flags) -#define SWIG_CheckImplicit(ty) SWIG_Python_CheckImplicit(ty) -#define SWIG_AcquirePtr(ptr, src) SWIG_Python_AcquirePtr(ptr, src) -#define swig_owntype int - -/* for raw packed data */ -#define SWIG_ConvertPacked(obj, ptr, sz, ty) SWIG_Python_ConvertPacked(obj, ptr, sz, ty) -#define SWIG_NewPackedObj(ptr, sz, type) SWIG_Python_NewPackedObj(ptr, sz, type) - -/* for class or struct pointers */ -#define SWIG_ConvertInstance(obj, pptr, type, flags) SWIG_ConvertPtr(obj, pptr, type, flags) -#define SWIG_NewInstanceObj(ptr, type, flags) SWIG_NewPointerObj(ptr, type, flags) - -/* for C or C++ function pointers */ -#define SWIG_ConvertFunctionPtr(obj, pptr, type) SWIG_Python_ConvertFunctionPtr(obj, pptr, type) -#define SWIG_NewFunctionPtrObj(ptr, type) SWIG_Python_NewPointerObj(ptr, type, 0) - -/* for C++ member pointers, ie, member methods */ -#define SWIG_ConvertMember(obj, ptr, sz, ty) SWIG_Python_ConvertPacked(obj, ptr, sz, ty) -#define SWIG_NewMemberObj(ptr, sz, type) SWIG_Python_NewPackedObj(ptr, sz, type) - - -/* Runtime API */ - -#define SWIG_GetModule(clientdata) SWIG_Python_GetModule() -#define SWIG_SetModule(clientdata, pointer) SWIG_Python_SetModule(pointer) -#define SWIG_NewClientData(obj) PySwigClientData_New(obj) - -#define SWIG_SetErrorObj SWIG_Python_SetErrorObj -#define SWIG_SetErrorMsg SWIG_Python_SetErrorMsg -#define SWIG_ErrorType(code) SWIG_Python_ErrorType(code) -#define SWIG_Error(code, msg) SWIG_Python_SetErrorMsg(SWIG_ErrorType(code), msg) -#define SWIG_fail goto fail - - -/* Runtime API implementation */ - -/* Error manipulation */ - -SWIGINTERN void -SWIG_Python_SetErrorObj(PyObject *errtype, PyObject *obj) { - SWIG_PYTHON_THREAD_BEGIN_BLOCK; - PyErr_SetObject(errtype, obj); - Py_DECREF(obj); - SWIG_PYTHON_THREAD_END_BLOCK; -} - -SWIGINTERN void -SWIG_Python_SetErrorMsg(PyObject *errtype, const char *msg) { - SWIG_PYTHON_THREAD_BEGIN_BLOCK; - PyErr_SetString(errtype, (char *) msg); - SWIG_PYTHON_THREAD_END_BLOCK; -} - -#define SWIG_Python_Raise(obj, type, desc) SWIG_Python_SetErrorObj(SWIG_Python_ExceptionType(desc), obj) - -/* Set a constant value */ - -SWIGINTERN void -SWIG_Python_SetConstant(PyObject *d, const char *name, PyObject *obj) { - PyDict_SetItemString(d, (char*) name, obj); - Py_DECREF(obj); -} - -/* Append a value to the result obj */ - -SWIGINTERN PyObject* -SWIG_Python_AppendOutput(PyObject* result, PyObject* obj) { -#if !defined(SWIG_PYTHON_OUTPUT_TUPLE) - if (!result) { - result = obj; - } else if (result == Py_None) { - Py_DECREF(result); - result = obj; - } else { - if (!PyList_Check(result)) { - PyObject *o2 = result; - result = PyList_New(1); - PyList_SetItem(result, 0, o2); - } - PyList_Append(result,obj); - Py_DECREF(obj); - } - return result; -#else - PyObject* o2; - PyObject* o3; - if (!result) { - result = obj; - } else if (result == Py_None) { - Py_DECREF(result); - result = obj; - } else { - if (!PyTuple_Check(result)) { - o2 = result; - result = PyTuple_New(1); - PyTuple_SET_ITEM(result, 0, o2); - } - o3 = PyTuple_New(1); - PyTuple_SET_ITEM(o3, 0, obj); - o2 = result; - result = PySequence_Concat(o2, o3); - Py_DECREF(o2); - Py_DECREF(o3); - } - return result; -#endif -} - -/* Unpack the argument tuple */ - -SWIGINTERN int -SWIG_Python_UnpackTuple(PyObject *args, const char *name, int min, int max, PyObject **objs) -{ - if (!args) { - if (!min && !max) { - return 1; - } else { - PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got none", - name, (min == max ? "" : "at least "), min); - return 0; - } - } - if (!PyTuple_Check(args)) { - PyErr_SetString(PyExc_SystemError, "UnpackTuple() argument list is not a tuple"); - return 0; - } else { - register int l = PyTuple_GET_SIZE(args); - if (l < min) { - PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d", - name, (min == max ? "" : "at least "), min, l); - return 0; - } else if (l > max) { - PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d", - name, (min == max ? "" : "at most "), max, l); - return 0; - } else { - register int i; - for (i = 0; i < l; ++i) { - objs[i] = PyTuple_GET_ITEM(args, i); - } - for (; l < max; ++l) { - objs[l] = 0; - } - return i + 1; - } - } -} - -/* A functor is a function object with one single object argument */ -#if PY_VERSION_HEX >= 0x02020000 -#define SWIG_Python_CallFunctor(functor, obj) PyObject_CallFunctionObjArgs(functor, obj, NULL); -#else -#define SWIG_Python_CallFunctor(functor, obj) PyObject_CallFunction(functor, "O", obj); -#endif - -/* - Helper for static pointer initialization for both C and C++ code, for example - static PyObject *SWIG_STATIC_POINTER(MyVar) = NewSomething(...); -*/ -#ifdef __cplusplus -#define SWIG_STATIC_POINTER(var) var -#else -#define SWIG_STATIC_POINTER(var) var = 0; if (!var) var -#endif - -/* ----------------------------------------------------------------------------- - * Pointer declarations - * ----------------------------------------------------------------------------- */ - -/* Flags for new pointer objects */ -#define SWIG_POINTER_NOSHADOW (SWIG_POINTER_OWN << 1) -#define SWIG_POINTER_NEW (SWIG_POINTER_NOSHADOW | SWIG_POINTER_OWN) - -#define SWIG_POINTER_IMPLICIT_CONV (SWIG_POINTER_DISOWN << 1) - -#ifdef __cplusplus -extern "C" { -#if 0 -} /* cc-mode */ -#endif -#endif - -/* How to access Py_None */ -#if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) -# ifndef SWIG_PYTHON_NO_BUILD_NONE -# ifndef SWIG_PYTHON_BUILD_NONE -# define SWIG_PYTHON_BUILD_NONE -# endif -# endif -#endif - -#ifdef SWIG_PYTHON_BUILD_NONE -# ifdef Py_None -# undef Py_None -# define Py_None SWIG_Py_None() -# endif -SWIGRUNTIMEINLINE PyObject * -_SWIG_Py_None(void) -{ - PyObject *none = Py_BuildValue((char*)""); - Py_DECREF(none); - return none; -} -SWIGRUNTIME PyObject * -SWIG_Py_None(void) -{ - static PyObject *SWIG_STATIC_POINTER(none) = _SWIG_Py_None(); - return none; -} -#endif - -/* The python void return value */ - -SWIGRUNTIMEINLINE PyObject * -SWIG_Py_Void(void) -{ - PyObject *none = Py_None; - Py_INCREF(none); - return none; -} - -/* PySwigClientData */ - -typedef struct { - PyObject *klass; - PyObject *newraw; - PyObject *newargs; - PyObject *destroy; - int delargs; - int implicitconv; -} PySwigClientData; - -SWIGRUNTIMEINLINE int -SWIG_Python_CheckImplicit(swig_type_info *ty) -{ - PySwigClientData *data = (PySwigClientData *)ty->clientdata; - return data ? data->implicitconv : 0; -} - -SWIGRUNTIMEINLINE PyObject * -SWIG_Python_ExceptionType(swig_type_info *desc) { - PySwigClientData *data = desc ? (PySwigClientData *) desc->clientdata : 0; - PyObject *klass = data ? data->klass : 0; - return (klass ? klass : PyExc_RuntimeError); -} - - -SWIGRUNTIME PySwigClientData * -PySwigClientData_New(PyObject* obj) -{ - if (!obj) { - return 0; - } else { - PySwigClientData *data = (PySwigClientData *)malloc(sizeof(PySwigClientData)); - /* the klass element */ - data->klass = obj; - Py_INCREF(data->klass); - /* the newraw method and newargs arguments used to create a new raw instance */ - if (PyClass_Check(obj)) { - data->newraw = 0; - data->newargs = obj; - Py_INCREF(obj); - } else { -#if (PY_VERSION_HEX < 0x02020000) - data->newraw = 0; -#else - data->newraw = PyObject_GetAttrString(data->klass, (char *)"__new__"); -#endif - if (data->newraw) { - Py_INCREF(data->newraw); - data->newargs = PyTuple_New(1); - PyTuple_SetItem(data->newargs, 0, obj); - } else { - data->newargs = obj; - } - Py_INCREF(data->newargs); - } - /* the destroy method, aka as the C++ delete method */ - data->destroy = PyObject_GetAttrString(data->klass, (char *)"__swig_destroy__"); - if (PyErr_Occurred()) { - PyErr_Clear(); - data->destroy = 0; - } - if (data->destroy) { - int flags; - Py_INCREF(data->destroy); - flags = PyCFunction_GET_FLAGS(data->destroy); -#ifdef METH_O - data->delargs = !(flags & (METH_O)); -#else - data->delargs = 0; -#endif - } else { - data->delargs = 0; - } - data->implicitconv = 0; - return data; - } -} - -SWIGRUNTIME void -PySwigClientData_Del(PySwigClientData* data) -{ - Py_XDECREF(data->newraw); - Py_XDECREF(data->newargs); - Py_XDECREF(data->destroy); -} - -/* =============== PySwigObject =====================*/ - -typedef struct { - PyObject_HEAD - void *ptr; - swig_type_info *ty; - int own; - PyObject *next; -} PySwigObject; - -SWIGRUNTIME PyObject * -PySwigObject_long(PySwigObject *v) -{ - return PyLong_FromVoidPtr(v->ptr); -} - -SWIGRUNTIME PyObject * -PySwigObject_format(const char* fmt, PySwigObject *v) -{ - PyObject *res = NULL; - PyObject *args = PyTuple_New(1); - if (args) { - if (PyTuple_SetItem(args, 0, PySwigObject_long(v)) == 0) { - PyObject *ofmt = PyString_FromString(fmt); - if (ofmt) { - res = PyString_Format(ofmt,args); - Py_DECREF(ofmt); - } - Py_DECREF(args); - } - } - return res; -} - -SWIGRUNTIME PyObject * -PySwigObject_oct(PySwigObject *v) -{ - return PySwigObject_format("%o",v); -} - -SWIGRUNTIME PyObject * -PySwigObject_hex(PySwigObject *v) -{ - return PySwigObject_format("%x",v); -} - -SWIGRUNTIME PyObject * -#ifdef METH_NOARGS -PySwigObject_repr(PySwigObject *v) -#else -PySwigObject_repr(PySwigObject *v, PyObject *args) -#endif -{ - const char *name = SWIG_TypePrettyName(v->ty); - PyObject *hex = PySwigObject_hex(v); - PyObject *repr = PyString_FromFormat("<Swig Object of type '%s' at 0x%s>", name, PyString_AsString(hex)); - Py_DECREF(hex); - if (v->next) { -#ifdef METH_NOARGS - PyObject *nrep = PySwigObject_repr((PySwigObject *)v->next); -#else - PyObject *nrep = PySwigObject_repr((PySwigObject *)v->next, args); -#endif - PyString_ConcatAndDel(&repr,nrep); - } - return repr; -} - -SWIGRUNTIME int -PySwigObject_print(PySwigObject *v, FILE *fp, int SWIGUNUSEDPARM(flags)) -{ -#ifdef METH_NOARGS - PyObject *repr = PySwigObject_repr(v); -#else - PyObject *repr = PySwigObject_repr(v, NULL); -#endif - if (repr) { - fputs(PyString_AsString(repr), fp); - Py_DECREF(repr); - return 0; - } else { - return 1; - } -} - -SWIGRUNTIME PyObject * -PySwigObject_str(PySwigObject *v) -{ - char result[SWIG_BUFFER_SIZE]; - return SWIG_PackVoidPtr(result, v->ptr, v->ty->name, sizeof(result)) ? - PyString_FromString(result) : 0; -} - -SWIGRUNTIME int -PySwigObject_compare(PySwigObject *v, PySwigObject *w) -{ - void *i = v->ptr; - void *j = w->ptr; - return (i < j) ? -1 : ((i > j) ? 1 : 0); -} - -SWIGRUNTIME PyTypeObject* _PySwigObject_type(void); - -SWIGRUNTIME PyTypeObject* -PySwigObject_type(void) { - static PyTypeObject *SWIG_STATIC_POINTER(type) = _PySwigObject_type(); - return type; -} - -SWIGRUNTIMEINLINE int -PySwigObject_Check(PyObject *op) { - return ((op)->ob_type == PySwigObject_type()) - || (strcmp((op)->ob_type->tp_name,"PySwigObject") == 0); -} - -SWIGRUNTIME PyObject * -PySwigObject_New(void *ptr, swig_type_info *ty, int own); - -SWIGRUNTIME void -PySwigObject_dealloc(PyObject *v) -{ - PySwigObject *sobj = (PySwigObject *) v; - PyObject *next = sobj->next; - if (sobj->own) { - swig_type_info *ty = sobj->ty; - PySwigClientData *data = ty ? (PySwigClientData *) ty->clientdata : 0; - PyObject *destroy = data ? data->destroy : 0; - if (destroy) { - /* destroy is always a VARARGS method */ - PyObject *res; - if (data->delargs) { - /* we need to create a temporal object to carry the destroy operation */ - PyObject *tmp = PySwigObject_New(sobj->ptr, ty, 0); - res = SWIG_Python_CallFunctor(destroy, tmp); - Py_DECREF(tmp); - } else { - PyCFunction meth = PyCFunction_GET_FUNCTION(destroy); - PyObject *mself = PyCFunction_GET_SELF(destroy); - res = ((*meth)(mself, v)); - } - Py_XDECREF(res); - } else { - const char *name = SWIG_TypePrettyName(ty); -#if !defined(SWIG_PYTHON_SILENT_MEMLEAK) - printf("swig/python detected a memory leak of type '%s', no destructor found.\n", name); -#endif - } - } - Py_XDECREF(next); - PyObject_DEL(v); -} - -SWIGRUNTIME PyObject* -PySwigObject_append(PyObject* v, PyObject* next) -{ - PySwigObject *sobj = (PySwigObject *) v; -#ifndef METH_O - PyObject *tmp = 0; - if (!PyArg_ParseTuple(next,(char *)"O:append", &tmp)) return NULL; - next = tmp; -#endif - if (!PySwigObject_Check(next)) { - return NULL; - } - sobj->next = next; - Py_INCREF(next); - return SWIG_Py_Void(); -} - -SWIGRUNTIME PyObject* -#ifdef METH_NOARGS -PySwigObject_next(PyObject* v) -#else -PySwigObject_next(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) -#endif -{ - PySwigObject *sobj = (PySwigObject *) v; - if (sobj->next) { - Py_INCREF(sobj->next); - return sobj->next; - } else { - return SWIG_Py_Void(); - } -} - -SWIGINTERN PyObject* -#ifdef METH_NOARGS -PySwigObject_disown(PyObject *v) -#else -PySwigObject_disown(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) -#endif -{ - PySwigObject *sobj = (PySwigObject *)v; - sobj->own = 0; - return SWIG_Py_Void(); -} - -SWIGINTERN PyObject* -#ifdef METH_NOARGS -PySwigObject_acquire(PyObject *v) -#else -PySwigObject_acquire(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) -#endif -{ - PySwigObject *sobj = (PySwigObject *)v; - sobj->own = SWIG_POINTER_OWN; - return SWIG_Py_Void(); -} - -SWIGINTERN PyObject* -PySwigObject_own(PyObject *v, PyObject *args) -{ - PyObject *val = 0; -#if (PY_VERSION_HEX < 0x02020000) - if (!PyArg_ParseTuple(args,(char *)"|O:own",&val)) -#else - if (!PyArg_UnpackTuple(args, (char *)"own", 0, 1, &val)) -#endif - { - return NULL; - } - else - { - PySwigObject *sobj = (PySwigObject *)v; - PyObject *obj = PyBool_FromLong(sobj->own); - if (val) { -#ifdef METH_NOARGS - if (PyObject_IsTrue(val)) { - PySwigObject_acquire(v); - } else { - PySwigObject_disown(v); - } -#else - if (PyObject_IsTrue(val)) { - PySwigObject_acquire(v,args); - } else { - PySwigObject_disown(v,args); - } -#endif - } - return obj; - } -} - -#ifdef METH_O -static PyMethodDef -swigobject_methods[] = { - {(char *)"disown", (PyCFunction)PySwigObject_disown, METH_NOARGS, (char *)"releases ownership of the pointer"}, - {(char *)"acquire", (PyCFunction)PySwigObject_acquire, METH_NOARGS, (char *)"aquires ownership of the pointer"}, - {(char *)"own", (PyCFunction)PySwigObject_own, METH_VARARGS, (char *)"returns/sets ownership of the pointer"}, - {(char *)"append", (PyCFunction)PySwigObject_append, METH_O, (char *)"appends another 'this' object"}, - {(char *)"next", (PyCFunction)PySwigObject_next, METH_NOARGS, (char *)"returns the next 'this' object"}, - {(char *)"__repr__",(PyCFunction)PySwigObject_repr, METH_NOARGS, (char *)"returns object representation"}, - {0, 0, 0, 0} -}; -#else -static PyMethodDef -swigobject_methods[] = { - {(char *)"disown", (PyCFunction)PySwigObject_disown, METH_VARARGS, (char *)"releases ownership of the pointer"}, - {(char *)"acquire", (PyCFunction)PySwigObject_acquire, METH_VARARGS, (char *)"aquires ownership of the pointer"}, - {(char *)"own", (PyCFunction)PySwigObject_own, METH_VARARGS, (char *)"returns/sets ownership of the pointer"}, - {(char *)"append", (PyCFunction)PySwigObject_append, METH_VARARGS, (char *)"appends another 'this' object"}, - {(char *)"next", (PyCFunction)PySwigObject_next, METH_VARARGS, (char *)"returns the next 'this' object"}, - {(char *)"__repr__",(PyCFunction)PySwigObject_repr, METH_VARARGS, (char *)"returns object representation"}, - {0, 0, 0, 0} -}; -#endif - -#if PY_VERSION_HEX < 0x02020000 -SWIGINTERN PyObject * -PySwigObject_getattr(PySwigObject *sobj,char *name) -{ - return Py_FindMethod(swigobject_methods, (PyObject *)sobj, name); -} -#endif - -SWIGRUNTIME PyTypeObject* -_PySwigObject_type(void) { - static char swigobject_doc[] = "Swig object carries a C/C++ instance pointer"; - - static PyNumberMethods PySwigObject_as_number = { - (binaryfunc)0, /*nb_add*/ - (binaryfunc)0, /*nb_subtract*/ - (binaryfunc)0, /*nb_multiply*/ - (binaryfunc)0, /*nb_divide*/ - (binaryfunc)0, /*nb_remainder*/ - (binaryfunc)0, /*nb_divmod*/ - (ternaryfunc)0,/*nb_power*/ - (unaryfunc)0, /*nb_negative*/ - (unaryfunc)0, /*nb_positive*/ - (unaryfunc)0, /*nb_absolute*/ - (inquiry)0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - (coercion)0, /*nb_coerce*/ - (unaryfunc)PySwigObject_long, /*nb_int*/ - (unaryfunc)PySwigObject_long, /*nb_long*/ - (unaryfunc)0, /*nb_float*/ - (unaryfunc)PySwigObject_oct, /*nb_oct*/ - (unaryfunc)PySwigObject_hex, /*nb_hex*/ -#if PY_VERSION_HEX >= 0x02020000 - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_inplace_true_divide */ -#elif PY_VERSION_HEX >= 0x02000000 - 0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_inplace_or */ -#endif - }; - - static PyTypeObject pyswigobject_type; - static int type_init = 0; - if (!type_init) { - const PyTypeObject tmp - = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ - (char *)"PySwigObject", /* tp_name */ - sizeof(PySwigObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PySwigObject_dealloc, /* tp_dealloc */ - (printfunc)PySwigObject_print, /* tp_print */ -#if PY_VERSION_HEX < 0x02020000 - (getattrfunc)PySwigObject_getattr, /* tp_getattr */ -#else - (getattrfunc)0, /* tp_getattr */ -#endif - (setattrfunc)0, /* tp_setattr */ - (cmpfunc)PySwigObject_compare, /* tp_compare */ - (reprfunc)PySwigObject_repr, /* tp_repr */ - &PySwigObject_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - (hashfunc)0, /* tp_hash */ - (ternaryfunc)0, /* tp_call */ - (reprfunc)PySwigObject_str, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - swigobject_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ -#if PY_VERSION_HEX >= 0x02020000 - 0, /* tp_iter */ - 0, /* tp_iternext */ - swigobject_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ -#endif -#if PY_VERSION_HEX >= 0x02030000 - 0, /* tp_del */ -#endif -#ifdef COUNT_ALLOCS - 0,0,0,0 /* tp_alloc -> tp_next */ -#endif - }; - pyswigobject_type = tmp; - pyswigobject_type.ob_type = &PyType_Type; - type_init = 1; - } - return &pyswigobject_type; -} - -SWIGRUNTIME PyObject * -PySwigObject_New(void *ptr, swig_type_info *ty, int own) -{ - PySwigObject *sobj = PyObject_NEW(PySwigObject, PySwigObject_type()); - if (sobj) { - sobj->ptr = ptr; - sobj->ty = ty; - sobj->own = own; - sobj->next = 0; - } - return (PyObject *)sobj; -} - -/* ----------------------------------------------------------------------------- - * Implements a simple Swig Packed type, and use it instead of string - * ----------------------------------------------------------------------------- */ - -typedef struct { - PyObject_HEAD - void *pack; - swig_type_info *ty; - size_t size; -} PySwigPacked; - -SWIGRUNTIME int -PySwigPacked_print(PySwigPacked *v, FILE *fp, int SWIGUNUSEDPARM(flags)) -{ - char result[SWIG_BUFFER_SIZE]; - fputs("<Swig Packed ", fp); - if (SWIG_PackDataName(result, v->pack, v->size, 0, sizeof(result))) { - fputs("at ", fp); - fputs(result, fp); - } - fputs(v->ty->name,fp); - fputs(">", fp); - return 0; -} - -SWIGRUNTIME PyObject * -PySwigPacked_repr(PySwigPacked *v) -{ - char result[SWIG_BUFFER_SIZE]; - if (SWIG_PackDataName(result, v->pack, v->size, 0, sizeof(result))) { - return PyString_FromFormat("<Swig Packed at %s%s>", result, v->ty->name); - } else { - return PyString_FromFormat("<Swig Packed %s>", v->ty->name); - } -} - -SWIGRUNTIME PyObject * -PySwigPacked_str(PySwigPacked *v) -{ - char result[SWIG_BUFFER_SIZE]; - if (SWIG_PackDataName(result, v->pack, v->size, 0, sizeof(result))){ - return PyString_FromFormat("%s%s", result, v->ty->name); - } else { - return PyString_FromString(v->ty->name); - } -} - -SWIGRUNTIME int -PySwigPacked_compare(PySwigPacked *v, PySwigPacked *w) -{ - size_t i = v->size; - size_t j = w->size; - int s = (i < j) ? -1 : ((i > j) ? 1 : 0); - return s ? s : strncmp((char *)v->pack, (char *)w->pack, 2*v->size); -} - -SWIGRUNTIME PyTypeObject* _PySwigPacked_type(void); - -SWIGRUNTIME PyTypeObject* -PySwigPacked_type(void) { - static PyTypeObject *SWIG_STATIC_POINTER(type) = _PySwigPacked_type(); - return type; -} - -SWIGRUNTIMEINLINE int -PySwigPacked_Check(PyObject *op) { - return ((op)->ob_type == _PySwigPacked_type()) - || (strcmp((op)->ob_type->tp_name,"PySwigPacked") == 0); -} - -SWIGRUNTIME void -PySwigPacked_dealloc(PyObject *v) -{ - if (PySwigPacked_Check(v)) { - PySwigPacked *sobj = (PySwigPacked *) v; - free(sobj->pack); - } - PyObject_DEL(v); -} - -SWIGRUNTIME PyTypeObject* -_PySwigPacked_type(void) { - static char swigpacked_doc[] = "Swig object carries a C/C++ instance pointer"; - static PyTypeObject pyswigpacked_type; - static int type_init = 0; - if (!type_init) { - const PyTypeObject tmp - = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ - (char *)"PySwigPacked", /* tp_name */ - sizeof(PySwigPacked), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PySwigPacked_dealloc, /* tp_dealloc */ - (printfunc)PySwigPacked_print, /* tp_print */ - (getattrfunc)0, /* tp_getattr */ - (setattrfunc)0, /* tp_setattr */ - (cmpfunc)PySwigPacked_compare, /* tp_compare */ - (reprfunc)PySwigPacked_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - (hashfunc)0, /* tp_hash */ - (ternaryfunc)0, /* tp_call */ - (reprfunc)PySwigPacked_str, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - swigpacked_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ -#if PY_VERSION_HEX >= 0x02020000 - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ -#endif -#if PY_VERSION_HEX >= 0x02030000 - 0, /* tp_del */ -#endif -#ifdef COUNT_ALLOCS - 0,0,0,0 /* tp_alloc -> tp_next */ -#endif - }; - pyswigpacked_type = tmp; - pyswigpacked_type.ob_type = &PyType_Type; - type_init = 1; - } - return &pyswigpacked_type; -} - -SWIGRUNTIME PyObject * -PySwigPacked_New(void *ptr, size_t size, swig_type_info *ty) -{ - PySwigPacked *sobj = PyObject_NEW(PySwigPacked, PySwigPacked_type()); - if (sobj) { - void *pack = malloc(size); - if (pack) { - memcpy(pack, ptr, size); - sobj->pack = pack; - sobj->ty = ty; - sobj->size = size; - } else { - PyObject_DEL((PyObject *) sobj); - sobj = 0; - } - } - return (PyObject *) sobj; -} - -SWIGRUNTIME swig_type_info * -PySwigPacked_UnpackData(PyObject *obj, void *ptr, size_t size) -{ - if (PySwigPacked_Check(obj)) { - PySwigPacked *sobj = (PySwigPacked *)obj; - if (sobj->size != size) return 0; - memcpy(ptr, sobj->pack, size); - return sobj->ty; - } else { - return 0; - } -} - -/* ----------------------------------------------------------------------------- - * pointers/data manipulation - * ----------------------------------------------------------------------------- */ - -SWIGRUNTIMEINLINE PyObject * -_SWIG_This(void) -{ - return PyString_FromString("this"); -} - -SWIGRUNTIME PyObject * -SWIG_This(void) -{ - static PyObject *SWIG_STATIC_POINTER(swig_this) = _SWIG_This(); - return swig_this; -} - -/* #define SWIG_PYTHON_SLOW_GETSET_THIS */ - -SWIGRUNTIME PySwigObject * -SWIG_Python_GetSwigThis(PyObject *pyobj) -{ - if (PySwigObject_Check(pyobj)) { - return (PySwigObject *) pyobj; - } else { - PyObject *obj = 0; -#if (!defined(SWIG_PYTHON_SLOW_GETSET_THIS) && (PY_VERSION_HEX >= 0x02030000)) - if (PyInstance_Check(pyobj)) { - obj = _PyInstance_Lookup(pyobj, SWIG_This()); - } else { - PyObject **dictptr = _PyObject_GetDictPtr(pyobj); - if (dictptr != NULL) { - PyObject *dict = *dictptr; - obj = dict ? PyDict_GetItem(dict, SWIG_This()) : 0; - } else { -#ifdef PyWeakref_CheckProxy - if (PyWeakref_CheckProxy(pyobj)) { - PyObject *wobj = PyWeakref_GET_OBJECT(pyobj); - return wobj ? SWIG_Python_GetSwigThis(wobj) : 0; - } -#endif - obj = PyObject_GetAttr(pyobj,SWIG_This()); - if (obj) { - Py_DECREF(obj); - } else { - if (PyErr_Occurred()) PyErr_Clear(); - return 0; - } - } - } -#else - obj = PyObject_GetAttr(pyobj,SWIG_This()); - if (obj) { - Py_DECREF(obj); - } else { - if (PyErr_Occurred()) PyErr_Clear(); - return 0; - } -#endif - if (obj && !PySwigObject_Check(obj)) { - /* a PyObject is called 'this', try to get the 'real this' - PySwigObject from it */ - return SWIG_Python_GetSwigThis(obj); - } - return (PySwigObject *)obj; - } -} - -/* Acquire a pointer value */ - -SWIGRUNTIME int -SWIG_Python_AcquirePtr(PyObject *obj, int own) { - if (own) { - PySwigObject *sobj = SWIG_Python_GetSwigThis(obj); - if (sobj) { - int oldown = sobj->own; - sobj->own = own; - return oldown; - } - } - return 0; -} - -/* Convert a pointer value */ - -SWIGRUNTIME int -SWIG_Python_ConvertPtrAndOwn(PyObject *obj, void **ptr, swig_type_info *ty, int flags, int *own) { - if (!obj) return SWIG_ERROR; - if (obj == Py_None) { - if (ptr) *ptr = 0; - return SWIG_OK; - } else { - PySwigObject *sobj = SWIG_Python_GetSwigThis(obj); - while (sobj) { - void *vptr = sobj->ptr; - if (ty) { - swig_type_info *to = sobj->ty; - if (to == ty) { - /* no type cast needed */ - if (ptr) *ptr = vptr; - break; - } else { - swig_cast_info *tc = SWIG_TypeCheck(to->name,ty); - if (!tc) { - sobj = (PySwigObject *)sobj->next; - } else { - if (ptr) *ptr = SWIG_TypeCast(tc,vptr); - break; - } - } - } else { - if (ptr) *ptr = vptr; - break; - } - } - if (sobj) { - if (own) *own = sobj->own; - if (flags & SWIG_POINTER_DISOWN) { - sobj->own = 0; - } - return SWIG_OK; - } else { - int res = SWIG_ERROR; - if (flags & SWIG_POINTER_IMPLICIT_CONV) { - PySwigClientData *data = ty ? (PySwigClientData *) ty->clientdata : 0; - if (data && !data->implicitconv) { - PyObject *klass = data->klass; - if (klass) { - PyObject *impconv; - data->implicitconv = 1; /* avoid recursion and call 'explicit' constructors*/ - impconv = SWIG_Python_CallFunctor(klass, obj); - data->implicitconv = 0; - if (PyErr_Occurred()) { - PyErr_Clear(); - impconv = 0; - } - if (impconv) { - PySwigObject *iobj = SWIG_Python_GetSwigThis(impconv); - if (iobj) { - void *vptr; - res = SWIG_Python_ConvertPtrAndOwn((PyObject*)iobj, &vptr, ty, 0, 0); - if (SWIG_IsOK(res)) { - if (ptr) { - *ptr = vptr; - /* transfer the ownership to 'ptr' */ - iobj->own = 0; - res = SWIG_AddCast(res); - res = SWIG_AddNewMask(res); - } else { - res = SWIG_AddCast(res); - } - } - } - Py_DECREF(impconv); - } - } - } - } - return res; - } - } -} - -/* Convert a function ptr value */ - -SWIGRUNTIME int -SWIG_Python_ConvertFunctionPtr(PyObject *obj, void **ptr, swig_type_info *ty) { - if (!PyCFunction_Check(obj)) { - return SWIG_ConvertPtr(obj, ptr, ty, 0); - } else { - void *vptr = 0; - - /* here we get the method pointer for callbacks */ - const char *doc = (((PyCFunctionObject *)obj) -> m_ml -> ml_doc); - const char *desc = doc ? strstr(doc, "swig_ptr: ") : 0; - if (desc) { - desc = ty ? SWIG_UnpackVoidPtr(desc + 10, &vptr, ty->name) : 0; - if (!desc) return SWIG_ERROR; - } - if (ty) { - swig_cast_info *tc = SWIG_TypeCheck(desc,ty); - if (!tc) return SWIG_ERROR; - *ptr = SWIG_TypeCast(tc,vptr); - } else { - *ptr = vptr; - } - return SWIG_OK; - } -} - -/* Convert a packed value value */ - -SWIGRUNTIME int -SWIG_Python_ConvertPacked(PyObject *obj, void *ptr, size_t sz, swig_type_info *ty) { - swig_type_info *to = PySwigPacked_UnpackData(obj, ptr, sz); - if (!to) return SWIG_ERROR; - if (ty) { - if (to != ty) { - /* check type cast? */ - swig_cast_info *tc = SWIG_TypeCheck(to->name,ty); - if (!tc) return SWIG_ERROR; - } - } - return SWIG_OK; -} - -/* ----------------------------------------------------------------------------- - * Create a new pointer object - * ----------------------------------------------------------------------------- */ - -/* - Create a new instance object, whitout calling __init__, and set the - 'this' attribute. -*/ - -SWIGRUNTIME PyObject* -SWIG_Python_NewShadowInstance(PySwigClientData *data, PyObject *swig_this) -{ -#if (PY_VERSION_HEX >= 0x02020000) - PyObject *inst = 0; - PyObject *newraw = data->newraw; - if (newraw) { - inst = PyObject_Call(newraw, data->newargs, NULL); - if (inst) { -#if !defined(SWIG_PYTHON_SLOW_GETSET_THIS) - PyObject **dictptr = _PyObject_GetDictPtr(inst); - if (dictptr != NULL) { - PyObject *dict = *dictptr; - if (dict == NULL) { - dict = PyDict_New(); - *dictptr = dict; - PyDict_SetItem(dict, SWIG_This(), swig_this); - } - } -#else - PyObject *key = SWIG_This(); - PyObject_SetAttr(inst, key, swig_this); -#endif - } - } else { - PyObject *dict = PyDict_New(); - PyDict_SetItem(dict, SWIG_This(), swig_this); - inst = PyInstance_NewRaw(data->newargs, dict); - Py_DECREF(dict); - } - return inst; -#else -#if (PY_VERSION_HEX >= 0x02010000) - PyObject *inst; - PyObject *dict = PyDict_New(); - PyDict_SetItem(dict, SWIG_This(), swig_this); - inst = PyInstance_NewRaw(data->newargs, dict); - Py_DECREF(dict); - return (PyObject *) inst; -#else - PyInstanceObject *inst = PyObject_NEW(PyInstanceObject, &PyInstance_Type); - if (inst == NULL) { - return NULL; - } - inst->in_class = (PyClassObject *)data->newargs; - Py_INCREF(inst->in_class); - inst->in_dict = PyDict_New(); - if (inst->in_dict == NULL) { - Py_DECREF(inst); - return NULL; - } -#ifdef Py_TPFLAGS_HAVE_WEAKREFS - inst->in_weakreflist = NULL; -#endif -#ifdef Py_TPFLAGS_GC - PyObject_GC_Init(inst); -#endif - PyDict_SetItem(inst->in_dict, SWIG_This(), swig_this); - return (PyObject *) inst; -#endif -#endif -} - -SWIGRUNTIME void -SWIG_Python_SetSwigThis(PyObject *inst, PyObject *swig_this) -{ - PyObject *dict; -#if (PY_VERSION_HEX >= 0x02020000) && !defined(SWIG_PYTHON_SLOW_GETSET_THIS) - PyObject **dictptr = _PyObject_GetDictPtr(inst); - if (dictptr != NULL) { - dict = *dictptr; - if (dict == NULL) { - dict = PyDict_New(); - *dictptr = dict; - } - PyDict_SetItem(dict, SWIG_This(), swig_this); - return; - } -#endif - dict = PyObject_GetAttrString(inst, (char*)"__dict__"); - PyDict_SetItem(dict, SWIG_This(), swig_this); - Py_DECREF(dict); -} - - -SWIGINTERN PyObject * -SWIG_Python_InitShadowInstance(PyObject *args) { - PyObject *obj[2]; - if (!SWIG_Python_UnpackTuple(args,(char*)"swiginit", 2, 2, obj)) { - return NULL; - } else { - PySwigObject *sthis = SWIG_Python_GetSwigThis(obj[0]); - if (sthis) { - PySwigObject_append((PyObject*) sthis, obj[1]); - } else { - SWIG_Python_SetSwigThis(obj[0], obj[1]); - } - return SWIG_Py_Void(); - } -} - -/* Create a new pointer object */ - -SWIGRUNTIME PyObject * -SWIG_Python_NewPointerObj(void *ptr, swig_type_info *type, int flags) { - if (!ptr) { - return SWIG_Py_Void(); - } else { - int own = (flags & SWIG_POINTER_OWN) ? SWIG_POINTER_OWN : 0; - PyObject *robj = PySwigObject_New(ptr, type, own); - PySwigClientData *clientdata = type ? (PySwigClientData *)(type->clientdata) : 0; - if (clientdata && !(flags & SWIG_POINTER_NOSHADOW)) { - PyObject *inst = SWIG_Python_NewShadowInstance(clientdata, robj); - if (inst) { - Py_DECREF(robj); - robj = inst; - } - } - return robj; - } -} - -/* Create a new packed object */ - -SWIGRUNTIMEINLINE PyObject * -SWIG_Python_NewPackedObj(void *ptr, size_t sz, swig_type_info *type) { - return ptr ? PySwigPacked_New((void *) ptr, sz, type) : SWIG_Py_Void(); -} - -/* -----------------------------------------------------------------------------* - * Get type list - * -----------------------------------------------------------------------------*/ - -#ifdef SWIG_LINK_RUNTIME -void *SWIG_ReturnGlobalTypeList(void *); -#endif - -SWIGRUNTIME swig_module_info * -SWIG_Python_GetModule(void) { - static void *type_pointer = (void *)0; - /* first check if module already created */ - if (!type_pointer) { -#ifdef SWIG_LINK_RUNTIME - type_pointer = SWIG_ReturnGlobalTypeList((void *)0); -#else - type_pointer = PyCObject_Import((char*)"swig_runtime_data" SWIG_RUNTIME_VERSION, - (char*)"type_pointer" SWIG_TYPE_TABLE_NAME); - if (PyErr_Occurred()) { - PyErr_Clear(); - type_pointer = (void *)0; - } -#endif - } - return (swig_module_info *) type_pointer; -} - -#if PY_MAJOR_VERSION < 2 -/* PyModule_AddObject function was introduced in Python 2.0. The following function - is copied out of Python/modsupport.c in python version 2.3.4 */ -SWIGINTERN int -PyModule_AddObject(PyObject *m, char *name, PyObject *o) -{ - PyObject *dict; - if (!PyModule_Check(m)) { - PyErr_SetString(PyExc_TypeError, - "PyModule_AddObject() needs module as first arg"); - return SWIG_ERROR; - } - if (!o) { - PyErr_SetString(PyExc_TypeError, - "PyModule_AddObject() needs non-NULL value"); - return SWIG_ERROR; - } - - dict = PyModule_GetDict(m); - if (dict == NULL) { - /* Internal error -- modules must have a dict! */ - PyErr_Format(PyExc_SystemError, "module '%s' has no __dict__", - PyModule_GetName(m)); - return SWIG_ERROR; - } - if (PyDict_SetItemString(dict, name, o)) - return SWIG_ERROR; - Py_DECREF(o); - return SWIG_OK; -} -#endif - -SWIGRUNTIME void -SWIG_Python_DestroyModule(void *vptr) -{ - swig_module_info *swig_module = (swig_module_info *) vptr; - swig_type_info **types = swig_module->types; - size_t i; - for (i =0; i < swig_module->size; ++i) { - swig_type_info *ty = types[i]; - if (ty->owndata) { - PySwigClientData *data = (PySwigClientData *) ty->clientdata; - if (data) PySwigClientData_Del(data); - } - } - Py_DECREF(SWIG_This()); -} - -SWIGRUNTIME void -SWIG_Python_SetModule(swig_module_info *swig_module) { - static PyMethodDef swig_empty_runtime_method_table[] = { {NULL, NULL, 0, NULL} };/* Sentinel */ - - PyObject *module = Py_InitModule((char*)"swig_runtime_data" SWIG_RUNTIME_VERSION, - swig_empty_runtime_method_table); - PyObject *pointer = PyCObject_FromVoidPtr((void *) swig_module, SWIG_Python_DestroyModule); - if (pointer && module) { - PyModule_AddObject(module, (char*)"type_pointer" SWIG_TYPE_TABLE_NAME, pointer); - } else { - Py_XDECREF(pointer); - } -} - -/* The python cached type query */ -SWIGRUNTIME PyObject * -SWIG_Python_TypeCache(void) { - static PyObject *SWIG_STATIC_POINTER(cache) = PyDict_New(); - return cache; -} - -SWIGRUNTIME swig_type_info * -SWIG_Python_TypeQuery(const char *type) -{ - PyObject *cache = SWIG_Python_TypeCache(); - PyObject *key = PyString_FromString(type); - PyObject *obj = PyDict_GetItem(cache, key); - swig_type_info *descriptor; - if (obj) { - descriptor = (swig_type_info *) PyCObject_AsVoidPtr(obj); - } else { - swig_module_info *swig_module = SWIG_Python_GetModule(); - descriptor = SWIG_TypeQueryModule(swig_module, swig_module, type); - if (descriptor) { - obj = PyCObject_FromVoidPtr(descriptor, NULL); - PyDict_SetItem(cache, key, obj); - Py_DECREF(obj); - } - } - Py_DECREF(key); - return descriptor; -} - -/* - For backward compatibility only -*/ -#define SWIG_POINTER_EXCEPTION 0 -#define SWIG_arg_fail(arg) SWIG_Python_ArgFail(arg) -#define SWIG_MustGetPtr(p, type, argnum, flags) SWIG_Python_MustGetPtr(p, type, argnum, flags) - -SWIGRUNTIME int -SWIG_Python_AddErrMesg(const char* mesg, int infront) -{ - if (PyErr_Occurred()) { - PyObject *type = 0; - PyObject *value = 0; - PyObject *traceback = 0; - PyErr_Fetch(&type, &value, &traceback); - if (value) { - PyObject *old_str = PyObject_Str(value); - Py_XINCREF(type); - PyErr_Clear(); - if (infront) { - PyErr_Format(type, "%s %s", mesg, PyString_AsString(old_str)); - } else { - PyErr_Format(type, "%s %s", PyString_AsString(old_str), mesg); - } - Py_DECREF(old_str); - } - return 1; - } else { - return 0; - } -} - -SWIGRUNTIME int -SWIG_Python_ArgFail(int argnum) -{ - if (PyErr_Occurred()) { - /* add information about failing argument */ - char mesg[256]; - PyOS_snprintf(mesg, sizeof(mesg), "argument number %d:", argnum); - return SWIG_Python_AddErrMesg(mesg, 1); - } else { - return 0; - } -} - -SWIGRUNTIMEINLINE const char * -PySwigObject_GetDesc(PyObject *self) -{ - PySwigObject *v = (PySwigObject *)self; - swig_type_info *ty = v ? v->ty : 0; - return ty ? ty->str : (char*)""; -} - -SWIGRUNTIME void -SWIG_Python_TypeError(const char *type, PyObject *obj) -{ - if (type) { -#if defined(SWIG_COBJECT_TYPES) - if (obj && PySwigObject_Check(obj)) { - const char *otype = (const char *) PySwigObject_GetDesc(obj); - if (otype) { - PyErr_Format(PyExc_TypeError, "a '%s' is expected, 'PySwigObject(%s)' is received", - type, otype); - return; - } - } else -#endif - { - const char *otype = (obj ? obj->ob_type->tp_name : 0); - if (otype) { - PyObject *str = PyObject_Str(obj); - const char *cstr = str ? PyString_AsString(str) : 0; - if (cstr) { - PyErr_Format(PyExc_TypeError, "a '%s' is expected, '%s(%s)' is received", - type, otype, cstr); - } else { - PyErr_Format(PyExc_TypeError, "a '%s' is expected, '%s' is received", - type, otype); - } - Py_XDECREF(str); - return; - } - } - PyErr_Format(PyExc_TypeError, "a '%s' is expected", type); - } else { - PyErr_Format(PyExc_TypeError, "unexpected type is received"); - } -} - - -/* Convert a pointer value, signal an exception on a type mismatch */ -SWIGRUNTIME void * -SWIG_Python_MustGetPtr(PyObject *obj, swig_type_info *ty, int argnum, int flags) { - void *result; - if (SWIG_Python_ConvertPtr(obj, &result, ty, flags) == -1) { - PyErr_Clear(); - if (flags & SWIG_POINTER_EXCEPTION) { - SWIG_Python_TypeError(SWIG_TypePrettyName(ty), obj); - SWIG_Python_ArgFail(argnum); - } - } - return result; -} - - -#ifdef __cplusplus -#if 0 -{ /* cc-mode */ -#endif -} -#endif - - - -#define SWIG_exception_fail(code, msg) do { SWIG_Error(code, msg); SWIG_fail; } while(0) - -#define SWIG_contract_assert(expr, msg) if (!(expr)) { SWIG_Error(SWIG_RuntimeError, msg); SWIG_fail; } else - - - -/* -------- TYPES TABLE (BEGIN) -------- */ - -#define SWIGTYPE_p_char swig_types[0] -#define SWIGTYPE_p_p_char swig_types[1] -#define SWIGTYPE_p_unsigned_int swig_types[2] -static swig_type_info *swig_types[4]; -static swig_module_info swig_module = {swig_types, 3, 0, 0, 0, 0}; -#define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name) -#define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name) - -/* -------- TYPES TABLE (END) -------- */ - -#if (PY_VERSION_HEX <= 0x02000000) -# if !defined(SWIG_PYTHON_CLASSIC) -# error "This python version requires swig to be run with the '-classic' option" -# endif -#endif - -/*----------------------------------------------- - @(target):= _xdelta3.so - ------------------------------------------------*/ -#define SWIG_init init_xdelta3 - -#define SWIG_name "_xdelta3" - -#define SWIGVERSION 0x010331 -#define SWIG_VERSION SWIGVERSION - - -#define SWIG_as_voidptr(a) (void *)((const void *)(a)) -#define SWIG_as_voidptrptr(a) ((void)SWIG_as_voidptr(*a),(void**)(a)) - - -#include "xdelta3.h" - -int xd3_main_cmdline (int ARGC, char **ARGV); - -#undef SWIG_init -#undef SWIG_name - -#define SWIG_init initxdelta3 -#define SWIG_name "xdelta3" - - - -SWIGINTERN swig_type_info* -SWIG_pchar_descriptor(void) -{ - static int init = 0; - static swig_type_info* info = 0; - if (!init) { - info = SWIG_TypeQuery("_p_char"); - init = 1; - } - return info; -} - - -SWIGINTERN int -SWIG_AsCharPtrAndSize(PyObject *obj, char** cptr, size_t* psize, int *alloc) -{ - if (PyString_Check(obj)) { - char *cstr; Py_ssize_t len; - PyString_AsStringAndSize(obj, &cstr, &len); - if (cptr) { - if (alloc) { - /* - In python the user should not be able to modify the inner - string representation. To warranty that, if you define - SWIG_PYTHON_SAFE_CSTRINGS, a new/copy of the python string - buffer is always returned. - - The default behavior is just to return the pointer value, - so, be careful. - */ -#if defined(SWIG_PYTHON_SAFE_CSTRINGS) - if (*alloc != SWIG_OLDOBJ) -#else - if (*alloc == SWIG_NEWOBJ) -#endif - { - *cptr = (char *)memcpy((char *)malloc((len + 1)*sizeof(char)), cstr, sizeof(char)*(len + 1)); - *alloc = SWIG_NEWOBJ; - } - else { - *cptr = cstr; - *alloc = SWIG_OLDOBJ; - } - } else { - *cptr = PyString_AsString(obj); - } - } - if (psize) *psize = len + 1; - return SWIG_OK; - } else { - swig_type_info* pchar_descriptor = SWIG_pchar_descriptor(); - if (pchar_descriptor) { - void* vptr = 0; - if (SWIG_ConvertPtr(obj, &vptr, pchar_descriptor, 0) == SWIG_OK) { - if (cptr) *cptr = (char *) vptr; - if (psize) *psize = vptr ? (strlen((char *)vptr) + 1) : 0; - if (alloc) *alloc = SWIG_OLDOBJ; - return SWIG_OK; - } - } - } - return SWIG_TypeError; -} - - -#include <limits.h> -#ifndef LLONG_MIN -# define LLONG_MIN LONG_LONG_MIN -#endif -#ifndef LLONG_MAX -# define LLONG_MAX LONG_LONG_MAX -#endif -#ifndef ULLONG_MAX -# define ULLONG_MAX ULONG_LONG_MAX -#endif - - -SWIGINTERN int -SWIG_AsVal_double (PyObject *obj, double *val) -{ - int res = SWIG_TypeError; - if (PyFloat_Check(obj)) { - if (val) *val = PyFloat_AsDouble(obj); - return SWIG_OK; - } else if (PyInt_Check(obj)) { - if (val) *val = PyInt_AsLong(obj); - return SWIG_OK; - } else if (PyLong_Check(obj)) { - double v = PyLong_AsDouble(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_OK; - } else { - PyErr_Clear(); - } - } -#ifdef SWIG_PYTHON_CAST_MODE - { - int dispatch = 0; - double d = PyFloat_AsDouble(obj); - if (!PyErr_Occurred()) { - if (val) *val = d; - return SWIG_AddCast(SWIG_OK); - } else { - PyErr_Clear(); - } - if (!dispatch) { - long v = PyLong_AsLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_AddCast(SWIG_AddCast(SWIG_OK)); - } else { - PyErr_Clear(); - } - } - } -#endif - return res; -} - - -#include <float.h> - - -#include <math.h> - - -SWIGINTERNINLINE int -SWIG_CanCastAsInteger(double *d, double min, double max) { - double x = *d; - if ((min <= x && x <= max)) { - double fx = floor(x); - double cx = ceil(x); - double rd = ((x - fx) < 0.5) ? fx : cx; /* simple rint */ - if ((errno == EDOM) || (errno == ERANGE)) { - errno = 0; - } else { - double summ, reps, diff; - if (rd < x) { - diff = x - rd; - } else if (rd > x) { - diff = rd - x; - } else { - return 1; - } - summ = rd + x; - reps = diff/summ; - if (reps < 8*DBL_EPSILON) { - *d = rd; - return 1; - } - } - } - return 0; -} - - -SWIGINTERN int -SWIG_AsVal_long (PyObject *obj, long* val) -{ - if (PyInt_Check(obj)) { - if (val) *val = PyInt_AsLong(obj); - return SWIG_OK; - } else if (PyLong_Check(obj)) { - long v = PyLong_AsLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_OK; - } else { - PyErr_Clear(); - } - } -#ifdef SWIG_PYTHON_CAST_MODE - { - int dispatch = 0; - long v = PyInt_AsLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_AddCast(SWIG_OK); - } else { - PyErr_Clear(); - } - if (!dispatch) { - double d; - int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); - if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, LONG_MIN, LONG_MAX)) { - if (val) *val = (long)(d); - return res; - } - } - } -#endif - return SWIG_TypeError; -} - - -SWIGINTERN int -SWIG_AsVal_int (PyObject * obj, int *val) -{ - long v; - int res = SWIG_AsVal_long (obj, &v); - if (SWIG_IsOK(res)) { - if ((v < INT_MIN || v > INT_MAX)) { - return SWIG_OverflowError; - } else { - if (val) *val = (int)(v); - } - } - return res; -} - - - - - -SWIGINTERN int -SWIG_AsVal_unsigned_SS_long (PyObject *obj, unsigned long *val) -{ - if (PyInt_Check(obj)) { - long v = PyInt_AsLong(obj); - if (v >= 0) { - if (val) *val = v; - return SWIG_OK; - } else { - return SWIG_OverflowError; - } - } else if (PyLong_Check(obj)) { - unsigned long v = PyLong_AsUnsignedLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_OK; - } else { - PyErr_Clear(); - } - } -#ifdef SWIG_PYTHON_CAST_MODE - { - int dispatch = 0; - unsigned long v = PyLong_AsUnsignedLong(obj); - if (!PyErr_Occurred()) { - if (val) *val = v; - return SWIG_AddCast(SWIG_OK); - } else { - PyErr_Clear(); - } - if (!dispatch) { - double d; - int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); - if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, 0, ULONG_MAX)) { - if (val) *val = (unsigned long)(d); - return res; - } - } - } -#endif - return SWIG_TypeError; -} - - -SWIGINTERN int -SWIG_AsVal_unsigned_SS_int (PyObject * obj, unsigned int *val) -{ - unsigned long v; - int res = SWIG_AsVal_unsigned_SS_long (obj, &v); - if (SWIG_IsOK(res)) { - if ((v > UINT_MAX)) { - return SWIG_OverflowError; - } else { - if (val) *val = (unsigned int)(v); - } - } - return res; -} - - -#define t_output_helper SWIG_Python_AppendOutput - - - #define SWIG_From_long PyInt_FromLong - - -SWIGINTERNINLINE PyObject * -SWIG_From_int (int value) -{ - return SWIG_From_long (value); -} - - -SWIGINTERN int -SWIG_AsArgcArgv(PyObject *input, - swig_type_info *ppchar_info, - size_t *argc, char ***argv, int *owner) -{ - void *vptr; - int res = SWIG_ConvertPtr(input, &vptr, ppchar_info, 0); - if (!SWIG_IsOK(res)) { - int list = 0; - PyErr_Clear(); - list = PyList_Check(input); - if (list || PyTuple_Check(input)) { - size_t i = 0; - size_t size = list ? PyList_Size(input) : PyTuple_Size(input); - if (argc) *argc = size; - if (argv) { - *argv = (char* *)malloc((size + 1)*sizeof(char*)); - for (; i < size; ++i) { - PyObject *obj = list ? PyList_GetItem(input,i) : PyTuple_GetItem(input,i); - char *cptr = 0; size_t sz = 0; int alloc = 0; - res = SWIG_AsCharPtrAndSize(obj, &cptr, &sz, &alloc); - if (SWIG_IsOK(res)) { - if (cptr && sz) { - (*argv)[i] = (alloc == SWIG_NEWOBJ) ? cptr : (char *)memcpy((char *)malloc((sz)*sizeof(char)), cptr, sizeof(char)*(sz)); - } else { - (*argv)[i] = 0; - } - } else { - return SWIG_TypeError; - } - } - (*argv)[i] = 0; - if (owner) *owner = 1; - } else { - for (; i < size; ++i) { - PyObject *obj = list ? PyList_GetItem(input,i) : PyTuple_GetItem(input,i); - res = SWIG_AsCharPtrAndSize(obj, 0, 0, 0); - if (!SWIG_IsOK(res)) return SWIG_TypeError; - } - if (owner) *owner = 0; - } - return SWIG_OK; - } else { - return SWIG_TypeError; - } - } else { - /* seems dangerous, but the user asked for it... */ - size_t i = 0; - if (argv) { while (*argv[i] != 0) ++i;} - if (argc) *argc = i; - if (owner) *owner = 0; - return SWIG_OK; - } -} - -#ifdef __cplusplus -extern "C" { -#endif -SWIGINTERN PyObject *_wrap_xd3_encode_memory(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - char *arg1 = (char *) 0 ; - unsigned int arg2 ; - char *arg3 = (char *) 0 ; - unsigned int arg4 ; - char *arg5 = (char *) 0 ; - unsigned int *arg6 = (unsigned int *) 0 ; - unsigned int arg7 ; - int arg8 ; - int result; - int res1 ; - char *buf1 = 0 ; - size_t size1 = 0 ; - int alloc1 = 0 ; - int res3 ; - char *buf3 = 0 ; - size_t size3 = 0 ; - int alloc3 = 0 ; - unsigned int alloc_size7 ; - int val8 ; - int ecode8 = 0 ; - PyObject * obj0 = 0 ; - PyObject * obj1 = 0 ; - PyObject * obj2 = 0 ; - PyObject * obj3 = 0 ; - - { - arg8 = 0; - } - { - - } - if (!PyArg_ParseTuple(args,(char *)"OOO|O:xd3_encode_memory",&obj0,&obj1,&obj2,&obj3)) SWIG_fail; - res1 = SWIG_AsCharPtrAndSize(obj0, &buf1, &size1, &alloc1); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "xd3_encode_memory" "', argument " "1"" of type '" "(const char *input, unsigned int input_size)""'"); - } - arg1 = (char *) buf1; - arg2 = (unsigned int) size1 - 1; - res3 = SWIG_AsCharPtrAndSize(obj1, &buf3, &size3, &alloc3); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "xd3_encode_memory" "', argument " "3"" of type '" "(const char *source, unsigned int source_size)""'"); - } - arg3 = (char *) buf3; - arg4 = (unsigned int) size3 - 1; - { - arg7 = alloc_size7 = PyInt_AsLong(obj2); - } - if (obj3) { - ecode8 = SWIG_AsVal_int(obj3, &val8); - if (!SWIG_IsOK(ecode8)) { - SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "xd3_encode_memory" "', argument " "8"" of type '" "int""'"); - } - arg8 = (int)(val8); - } - { - // alloc_size input is #7th position in xd3_xxcode_memory() - arg5 = malloc(alloc_size7); - arg6 = &alloc_size7; - } - result = (int)xd3_encode_memory((char const *)arg1,arg2,(char const *)arg3,arg4,arg5,arg6,arg7,arg8); - resultobj = SWIG_From_int((int)(result)); - { - if (result == 0) { - PyObject *o; - // alloc_size7 now carries actual size - o = PyString_FromStringAndSize(arg5,alloc_size7); - resultobj = t_output_helper(resultobj,o); - } else { - resultobj = t_output_helper(resultobj,Py_None); - } - free(arg5); - } - if (alloc1 == SWIG_NEWOBJ) free((char*)buf1); - if (alloc3 == SWIG_NEWOBJ) free((char*)buf3); - return resultobj; -fail: - if (alloc1 == SWIG_NEWOBJ) free((char*)buf1); - if (alloc3 == SWIG_NEWOBJ) free((char*)buf3); - return NULL; -} - - -SWIGINTERN PyObject *_wrap_xd3_decode_memory(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - char *arg1 = (char *) 0 ; - unsigned int arg2 ; - char *arg3 = (char *) 0 ; - unsigned int arg4 ; - char *arg5 = (char *) 0 ; - unsigned int *arg6 = (unsigned int *) 0 ; - unsigned int arg7 ; - int arg8 ; - int result; - int res1 ; - char *buf1 = 0 ; - size_t size1 = 0 ; - int alloc1 = 0 ; - int res3 ; - char *buf3 = 0 ; - size_t size3 = 0 ; - int alloc3 = 0 ; - unsigned int alloc_size7 ; - int val8 ; - int ecode8 = 0 ; - PyObject * obj0 = 0 ; - PyObject * obj1 = 0 ; - PyObject * obj2 = 0 ; - PyObject * obj3 = 0 ; - - { - arg8 = 0; - } - { - - } - if (!PyArg_ParseTuple(args,(char *)"OOO|O:xd3_decode_memory",&obj0,&obj1,&obj2,&obj3)) SWIG_fail; - res1 = SWIG_AsCharPtrAndSize(obj0, &buf1, &size1, &alloc1); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "xd3_decode_memory" "', argument " "1"" of type '" "(const char *input, unsigned int input_size)""'"); - } - arg1 = (char *) buf1; - arg2 = (unsigned int) size1 - 1; - res3 = SWIG_AsCharPtrAndSize(obj1, &buf3, &size3, &alloc3); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "xd3_decode_memory" "', argument " "3"" of type '" "(const char *source, unsigned int source_size)""'"); - } - arg3 = (char *) buf3; - arg4 = (unsigned int) size3 - 1; - { - arg7 = alloc_size7 = PyInt_AsLong(obj2); - } - if (obj3) { - ecode8 = SWIG_AsVal_int(obj3, &val8); - if (!SWIG_IsOK(ecode8)) { - SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "xd3_decode_memory" "', argument " "8"" of type '" "int""'"); - } - arg8 = (int)(val8); - } - { - // alloc_size input is #7th position in xd3_xxcode_memory() - arg5 = malloc(alloc_size7); - arg6 = &alloc_size7; - } - result = (int)xd3_decode_memory((char const *)arg1,arg2,(char const *)arg3,arg4,arg5,arg6,arg7,arg8); - resultobj = SWIG_From_int((int)(result)); - { - if (result == 0) { - PyObject *o; - // alloc_size7 now carries actual size - o = PyString_FromStringAndSize(arg5,alloc_size7); - resultobj = t_output_helper(resultobj,o); - } else { - resultobj = t_output_helper(resultobj,Py_None); - } - free(arg5); - } - if (alloc1 == SWIG_NEWOBJ) free((char*)buf1); - if (alloc3 == SWIG_NEWOBJ) free((char*)buf3); - return resultobj; -fail: - if (alloc1 == SWIG_NEWOBJ) free((char*)buf1); - if (alloc3 == SWIG_NEWOBJ) free((char*)buf3); - return NULL; -} - - -SWIGINTERN PyObject *_wrap_xd3_main_cmdline(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *resultobj = 0; - int arg1 ; - char **arg2 = (char **) 0 ; - int result; - int res1 ; - char **argv1 = 0 ; - size_t argc1 = 0 ; - int owner1 = 0 ; - PyObject * obj0 = 0 ; - - if (!PyArg_ParseTuple(args,(char *)"O:xd3_main_cmdline",&obj0)) SWIG_fail; - res1 = SWIG_AsArgcArgv(obj0, SWIGTYPE_p_p_char, &argc1, &argv1, &owner1); - if (!SWIG_IsOK(res1)) { - arg1 = 0; arg2 = 0; - SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "xd3_main_cmdline" "', argument " "1"" of type '" "int ARGC, char **ARGV""'"); - } else { - arg1 = (int)(argc1); - arg2 = (char **)(argv1); - } - result = (int)xd3_main_cmdline(arg1,arg2); - resultobj = SWIG_From_int((int)(result)); - if (owner1) { - size_t i = argc1; - while (i) { - free((char*)argv1[--i]); - } - free((char*)argv1); - } - return resultobj; -fail: - if (owner1) { - size_t i = argc1; - while (i) { - free((char*)argv1[--i]); - } - free((char*)argv1); - } - return NULL; -} - - -static PyMethodDef SwigMethods[] = { - { (char *)"xd3_encode_memory", _wrap_xd3_encode_memory, METH_VARARGS, NULL}, - { (char *)"xd3_decode_memory", _wrap_xd3_decode_memory, METH_VARARGS, NULL}, - { (char *)"xd3_main_cmdline", _wrap_xd3_main_cmdline, METH_VARARGS, NULL}, - { NULL, NULL, 0, NULL } -}; - - -/* -------- TYPE CONVERSION AND EQUIVALENCE RULES (BEGIN) -------- */ - -static swig_type_info _swigt__p_char = {"_p_char", "char *", 0, 0, (void*)0, 0}; -static swig_type_info _swigt__p_p_char = {"_p_p_char", "char **", 0, 0, (void*)0, 0}; -static swig_type_info _swigt__p_unsigned_int = {"_p_unsigned_int", "unsigned int *", 0, 0, (void*)0, 0}; - -static swig_type_info *swig_type_initial[] = { - &_swigt__p_char, - &_swigt__p_p_char, - &_swigt__p_unsigned_int, -}; - -static swig_cast_info _swigc__p_char[] = { {&_swigt__p_char, 0, 0, 0},{0, 0, 0, 0}}; -static swig_cast_info _swigc__p_p_char[] = { {&_swigt__p_p_char, 0, 0, 0},{0, 0, 0, 0}}; -static swig_cast_info _swigc__p_unsigned_int[] = { {&_swigt__p_unsigned_int, 0, 0, 0},{0, 0, 0, 0}}; - -static swig_cast_info *swig_cast_initial[] = { - _swigc__p_char, - _swigc__p_p_char, - _swigc__p_unsigned_int, -}; - - -/* -------- TYPE CONVERSION AND EQUIVALENCE RULES (END) -------- */ - -static swig_const_info swig_const_table[] = { -{0, 0, 0, 0.0, 0, 0}}; - -#ifdef __cplusplus -} -#endif -/* ----------------------------------------------------------------------------- - * Type initialization: - * This problem is tough by the requirement that no dynamic - * memory is used. Also, since swig_type_info structures store pointers to - * swig_cast_info structures and swig_cast_info structures store pointers back - * to swig_type_info structures, we need some lookup code at initialization. - * The idea is that swig generates all the structures that are needed. - * The runtime then collects these partially filled structures. - * The SWIG_InitializeModule function takes these initial arrays out of - * swig_module, and does all the lookup, filling in the swig_module.types - * array with the correct data and linking the correct swig_cast_info - * structures together. - * - * The generated swig_type_info structures are assigned staticly to an initial - * array. We just loop through that array, and handle each type individually. - * First we lookup if this type has been already loaded, and if so, use the - * loaded structure instead of the generated one. Then we have to fill in the - * cast linked list. The cast data is initially stored in something like a - * two-dimensional array. Each row corresponds to a type (there are the same - * number of rows as there are in the swig_type_initial array). Each entry in - * a column is one of the swig_cast_info structures for that type. - * The cast_initial array is actually an array of arrays, because each row has - * a variable number of columns. So to actually build the cast linked list, - * we find the array of casts associated with the type, and loop through it - * adding the casts to the list. The one last trick we need to do is making - * sure the type pointer in the swig_cast_info struct is correct. - * - * First off, we lookup the cast->type name to see if it is already loaded. - * There are three cases to handle: - * 1) If the cast->type has already been loaded AND the type we are adding - * casting info to has not been loaded (it is in this module), THEN we - * replace the cast->type pointer with the type pointer that has already - * been loaded. - * 2) If BOTH types (the one we are adding casting info to, and the - * cast->type) are loaded, THEN the cast info has already been loaded by - * the previous module so we just ignore it. - * 3) Finally, if cast->type has not already been loaded, then we add that - * swig_cast_info to the linked list (because the cast->type) pointer will - * be correct. - * ----------------------------------------------------------------------------- */ - -#ifdef __cplusplus -extern "C" { -#if 0 -} /* c-mode */ -#endif -#endif - -#if 0 -#define SWIGRUNTIME_DEBUG -#endif - - -SWIGRUNTIME void -SWIG_InitializeModule(void *clientdata) { - size_t i; - swig_module_info *module_head, *iter; - int found; - - clientdata = clientdata; - - /* check to see if the circular list has been setup, if not, set it up */ - if (swig_module.next==0) { - /* Initialize the swig_module */ - swig_module.type_initial = swig_type_initial; - swig_module.cast_initial = swig_cast_initial; - swig_module.next = &swig_module; - } - - /* Try and load any already created modules */ - module_head = SWIG_GetModule(clientdata); - if (!module_head) { - /* This is the first module loaded for this interpreter */ - /* so set the swig module into the interpreter */ - SWIG_SetModule(clientdata, &swig_module); - module_head = &swig_module; - } else { - /* the interpreter has loaded a SWIG module, but has it loaded this one? */ - found=0; - iter=module_head; - do { - if (iter==&swig_module) { - found=1; - break; - } - iter=iter->next; - } while (iter!= module_head); - - /* if the is found in the list, then all is done and we may leave */ - if (found) return; - /* otherwise we must add out module into the list */ - swig_module.next = module_head->next; - module_head->next = &swig_module; - } - - /* Now work on filling in swig_module.types */ -#ifdef SWIGRUNTIME_DEBUG - printf("SWIG_InitializeModule: size %d\n", swig_module.size); -#endif - for (i = 0; i < swig_module.size; ++i) { - swig_type_info *type = 0; - swig_type_info *ret; - swig_cast_info *cast; - -#ifdef SWIGRUNTIME_DEBUG - printf("SWIG_InitializeModule: type %d %s\n", i, swig_module.type_initial[i]->name); -#endif - - /* if there is another module already loaded */ - if (swig_module.next != &swig_module) { - type = SWIG_MangledTypeQueryModule(swig_module.next, &swig_module, swig_module.type_initial[i]->name); - } - if (type) { - /* Overwrite clientdata field */ -#ifdef SWIGRUNTIME_DEBUG - printf("SWIG_InitializeModule: found type %s\n", type->name); -#endif - if (swig_module.type_initial[i]->clientdata) { - type->clientdata = swig_module.type_initial[i]->clientdata; -#ifdef SWIGRUNTIME_DEBUG - printf("SWIG_InitializeModule: found and overwrite type %s \n", type->name); -#endif - } - } else { - type = swig_module.type_initial[i]; - } - - /* Insert casting types */ - cast = swig_module.cast_initial[i]; - while (cast->type) { - /* Don't need to add information already in the list */ - ret = 0; -#ifdef SWIGRUNTIME_DEBUG - printf("SWIG_InitializeModule: look cast %s\n", cast->type->name); -#endif - if (swig_module.next != &swig_module) { - ret = SWIG_MangledTypeQueryModule(swig_module.next, &swig_module, cast->type->name); -#ifdef SWIGRUNTIME_DEBUG - if (ret) printf("SWIG_InitializeModule: found cast %s\n", ret->name); -#endif - } - if (ret) { - if (type == swig_module.type_initial[i]) { -#ifdef SWIGRUNTIME_DEBUG - printf("SWIG_InitializeModule: skip old type %s\n", ret->name); -#endif - cast->type = ret; - ret = 0; - } else { - /* Check for casting already in the list */ - swig_cast_info *ocast = SWIG_TypeCheck(ret->name, type); -#ifdef SWIGRUNTIME_DEBUG - if (ocast) printf("SWIG_InitializeModule: skip old cast %s\n", ret->name); -#endif - if (!ocast) ret = 0; - } - } - - if (!ret) { -#ifdef SWIGRUNTIME_DEBUG - printf("SWIG_InitializeModule: adding cast %s\n", cast->type->name); -#endif - if (type->cast) { - type->cast->prev = cast; - cast->next = type->cast; - } - type->cast = cast; - } - cast++; - } - /* Set entry in modules->types array equal to the type */ - swig_module.types[i] = type; - } - swig_module.types[i] = 0; - -#ifdef SWIGRUNTIME_DEBUG - printf("**** SWIG_InitializeModule: Cast List ******\n"); - for (i = 0; i < swig_module.size; ++i) { - int j = 0; - swig_cast_info *cast = swig_module.cast_initial[i]; - printf("SWIG_InitializeModule: type %d %s\n", i, swig_module.type_initial[i]->name); - while (cast->type) { - printf("SWIG_InitializeModule: cast type %s\n", cast->type->name); - cast++; - ++j; - } - printf("---- Total casts: %d\n",j); - } - printf("**** SWIG_InitializeModule: Cast List ******\n"); -#endif -} - -/* This function will propagate the clientdata field of type to -* any new swig_type_info structures that have been added into the list -* of equivalent types. It is like calling -* SWIG_TypeClientData(type, clientdata) a second time. -*/ -SWIGRUNTIME void -SWIG_PropagateClientData(void) { - size_t i; - swig_cast_info *equiv; - static int init_run = 0; - - if (init_run) return; - init_run = 1; - - for (i = 0; i < swig_module.size; i++) { - if (swig_module.types[i]->clientdata) { - equiv = swig_module.types[i]->cast; - while (equiv) { - if (!equiv->converter) { - if (equiv->type && !equiv->type->clientdata) - SWIG_TypeClientData(equiv->type, swig_module.types[i]->clientdata); - } - equiv = equiv->next; - } - } - } -} - -#ifdef __cplusplus -#if 0 -{ - /* c-mode */ -#endif -} -#endif - - - -#ifdef __cplusplus -extern "C" { -#endif - - /* Python-specific SWIG API */ -#define SWIG_newvarlink() SWIG_Python_newvarlink() -#define SWIG_addvarlink(p, name, get_attr, set_attr) SWIG_Python_addvarlink(p, name, get_attr, set_attr) -#define SWIG_InstallConstants(d, constants) SWIG_Python_InstallConstants(d, constants) - - /* ----------------------------------------------------------------------------- - * global variable support code. - * ----------------------------------------------------------------------------- */ - - typedef struct swig_globalvar { - char *name; /* Name of global variable */ - PyObject *(*get_attr)(void); /* Return the current value */ - int (*set_attr)(PyObject *); /* Set the value */ - struct swig_globalvar *next; - } swig_globalvar; - - typedef struct swig_varlinkobject { - PyObject_HEAD - swig_globalvar *vars; - } swig_varlinkobject; - - SWIGINTERN PyObject * - swig_varlink_repr(swig_varlinkobject *SWIGUNUSEDPARM(v)) { - return PyString_FromString("<Swig global variables>"); - } - - SWIGINTERN PyObject * - swig_varlink_str(swig_varlinkobject *v) { - PyObject *str = PyString_FromString("("); - swig_globalvar *var; - for (var = v->vars; var; var=var->next) { - PyString_ConcatAndDel(&str,PyString_FromString(var->name)); - if (var->next) PyString_ConcatAndDel(&str,PyString_FromString(", ")); - } - PyString_ConcatAndDel(&str,PyString_FromString(")")); - return str; - } - - SWIGINTERN int - swig_varlink_print(swig_varlinkobject *v, FILE *fp, int SWIGUNUSEDPARM(flags)) { - PyObject *str = swig_varlink_str(v); - fprintf(fp,"Swig global variables "); - fprintf(fp,"%s\n", PyString_AsString(str)); - Py_DECREF(str); - return 0; - } - - SWIGINTERN void - swig_varlink_dealloc(swig_varlinkobject *v) { - swig_globalvar *var = v->vars; - while (var) { - swig_globalvar *n = var->next; - free(var->name); - free(var); - var = n; - } - } - - SWIGINTERN PyObject * - swig_varlink_getattr(swig_varlinkobject *v, char *n) { - PyObject *res = NULL; - swig_globalvar *var = v->vars; - while (var) { - if (strcmp(var->name,n) == 0) { - res = (*var->get_attr)(); - break; - } - var = var->next; - } - if (res == NULL && !PyErr_Occurred()) { - PyErr_SetString(PyExc_NameError,"Unknown C global variable"); - } - return res; - } - - SWIGINTERN int - swig_varlink_setattr(swig_varlinkobject *v, char *n, PyObject *p) { - int res = 1; - swig_globalvar *var = v->vars; - while (var) { - if (strcmp(var->name,n) == 0) { - res = (*var->set_attr)(p); - break; - } - var = var->next; - } - if (res == 1 && !PyErr_Occurred()) { - PyErr_SetString(PyExc_NameError,"Unknown C global variable"); - } - return res; - } - - SWIGINTERN PyTypeObject* - swig_varlink_type(void) { - static char varlink__doc__[] = "Swig var link object"; - static PyTypeObject varlink_type; - static int type_init = 0; - if (!type_init) { - const PyTypeObject tmp - = { - PyObject_HEAD_INIT(NULL) - 0, /* Number of items in variable part (ob_size) */ - (char *)"swigvarlink", /* Type name (tp_name) */ - sizeof(swig_varlinkobject), /* Basic size (tp_basicsize) */ - 0, /* Itemsize (tp_itemsize) */ - (destructor) swig_varlink_dealloc, /* Deallocator (tp_dealloc) */ - (printfunc) swig_varlink_print, /* Print (tp_print) */ - (getattrfunc) swig_varlink_getattr, /* get attr (tp_getattr) */ - (setattrfunc) swig_varlink_setattr, /* Set attr (tp_setattr) */ - 0, /* tp_compare */ - (reprfunc) swig_varlink_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)swig_varlink_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - 0, /* tp_flags */ - varlink__doc__, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ -#if PY_VERSION_HEX >= 0x02020000 - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* tp_iter -> tp_weaklist */ -#endif -#if PY_VERSION_HEX >= 0x02030000 - 0, /* tp_del */ -#endif -#ifdef COUNT_ALLOCS - 0,0,0,0 /* tp_alloc -> tp_next */ -#endif - }; - varlink_type = tmp; - varlink_type.ob_type = &PyType_Type; - type_init = 1; - } - return &varlink_type; - } - - /* Create a variable linking object for use later */ - SWIGINTERN PyObject * - SWIG_Python_newvarlink(void) { - swig_varlinkobject *result = PyObject_NEW(swig_varlinkobject, swig_varlink_type()); - if (result) { - result->vars = 0; - } - return ((PyObject*) result); - } - - SWIGINTERN void - SWIG_Python_addvarlink(PyObject *p, char *name, PyObject *(*get_attr)(void), int (*set_attr)(PyObject *p)) { - swig_varlinkobject *v = (swig_varlinkobject *) p; - swig_globalvar *gv = (swig_globalvar *) malloc(sizeof(swig_globalvar)); - if (gv) { - size_t size = strlen(name)+1; - gv->name = (char *)malloc(size); - if (gv->name) { - strncpy(gv->name,name,size); - gv->get_attr = get_attr; - gv->set_attr = set_attr; - gv->next = v->vars; - } - } - v->vars = gv; - } - - SWIGINTERN PyObject * - SWIG_globals(void) { - static PyObject *_SWIG_globals = 0; - if (!_SWIG_globals) _SWIG_globals = SWIG_newvarlink(); - return _SWIG_globals; - } - - /* ----------------------------------------------------------------------------- - * constants/methods manipulation - * ----------------------------------------------------------------------------- */ - - /* Install Constants */ - SWIGINTERN void - SWIG_Python_InstallConstants(PyObject *d, swig_const_info constants[]) { - PyObject *obj = 0; - size_t i; - for (i = 0; constants[i].type; ++i) { - switch(constants[i].type) { - case SWIG_PY_POINTER: - obj = SWIG_NewPointerObj(constants[i].pvalue, *(constants[i]).ptype,0); - break; - case SWIG_PY_BINARY: - obj = SWIG_NewPackedObj(constants[i].pvalue, constants[i].lvalue, *(constants[i].ptype)); - break; - default: - obj = 0; - break; - } - if (obj) { - PyDict_SetItemString(d, constants[i].name, obj); - Py_DECREF(obj); - } - } - } - - /* -----------------------------------------------------------------------------*/ - /* Fix SwigMethods to carry the callback ptrs when needed */ - /* -----------------------------------------------------------------------------*/ - - SWIGINTERN void - SWIG_Python_FixMethods(PyMethodDef *methods, - swig_const_info *const_table, - swig_type_info **types, - swig_type_info **types_initial) { - size_t i; - for (i = 0; methods[i].ml_name; ++i) { - const char *c = methods[i].ml_doc; - if (c && (c = strstr(c, "swig_ptr: "))) { - int j; - swig_const_info *ci = 0; - const char *name = c + 10; - for (j = 0; const_table[j].type; ++j) { - if (strncmp(const_table[j].name, name, - strlen(const_table[j].name)) == 0) { - ci = &(const_table[j]); - break; - } - } - if (ci) { - size_t shift = (ci->ptype) - types; - swig_type_info *ty = types_initial[shift]; - size_t ldoc = (c - methods[i].ml_doc); - size_t lptr = strlen(ty->name)+2*sizeof(void*)+2; - char *ndoc = (char*)malloc(ldoc + lptr + 10); - if (ndoc) { - char *buff = ndoc; - void *ptr = (ci->type == SWIG_PY_POINTER) ? ci->pvalue : 0; - if (ptr) { - strncpy(buff, methods[i].ml_doc, ldoc); - buff += ldoc; - strncpy(buff, "swig_ptr: ", 10); - buff += 10; - SWIG_PackVoidPtr(buff, ptr, ty->name, lptr); - methods[i].ml_doc = ndoc; - } - } - } - } - } - } - -#ifdef __cplusplus -} -#endif - -/* -----------------------------------------------------------------------------* - * Partial Init method - * -----------------------------------------------------------------------------*/ - -#ifdef __cplusplus -extern "C" -#endif -SWIGEXPORT void SWIG_init(void) { - PyObject *m, *d; - - /* Fix SwigMethods to carry the callback ptrs when needed */ - SWIG_Python_FixMethods(SwigMethods, swig_const_table, swig_types, swig_type_initial); - - m = Py_InitModule((char *) SWIG_name, SwigMethods); - d = PyModule_GetDict(m); - - SWIG_InitializeModule(0); - SWIG_InstallConstants(d,swig_const_table); - - - SWIG_Python_SetConstant(d, "XD3_SEC_DJW",SWIG_From_int((int)(XD3_SEC_DJW))); - SWIG_Python_SetConstant(d, "XD3_SEC_FGK",SWIG_From_int((int)(XD3_SEC_FGK))); - SWIG_Python_SetConstant(d, "XD3_SEC_NODATA",SWIG_From_int((int)(XD3_SEC_NODATA))); - SWIG_Python_SetConstant(d, "XD3_SEC_NOINST",SWIG_From_int((int)(XD3_SEC_NOINST))); - SWIG_Python_SetConstant(d, "XD3_SEC_NOADDR",SWIG_From_int((int)(XD3_SEC_NOADDR))); - SWIG_Python_SetConstant(d, "XD3_ADLER32",SWIG_From_int((int)(XD3_ADLER32))); - SWIG_Python_SetConstant(d, "XD3_ADLER32_NOVER",SWIG_From_int((int)(XD3_ADLER32_NOVER))); - SWIG_Python_SetConstant(d, "XD3_ALT_CODE_TABLE",SWIG_From_int((int)(XD3_ALT_CODE_TABLE))); - SWIG_Python_SetConstant(d, "XD3_NOCOMPRESS",SWIG_From_int((int)(XD3_NOCOMPRESS))); - SWIG_Python_SetConstant(d, "XD3_BEGREEDY",SWIG_From_int((int)(XD3_BEGREEDY))); - SWIG_Python_SetConstant(d, "XD3_COMPLEVEL_SHIFT",SWIG_From_int((int)(XD3_COMPLEVEL_SHIFT))); - SWIG_Python_SetConstant(d, "XD3_COMPLEVEL_MASK",SWIG_From_int((int)(XD3_COMPLEVEL_MASK))); - SWIG_Python_SetConstant(d, "XD3_COMPLEVEL_1",SWIG_From_int((int)(XD3_COMPLEVEL_1))); - SWIG_Python_SetConstant(d, "XD3_COMPLEVEL_3",SWIG_From_int((int)(XD3_COMPLEVEL_3))); - SWIG_Python_SetConstant(d, "XD3_COMPLEVEL_6",SWIG_From_int((int)(XD3_COMPLEVEL_6))); - SWIG_Python_SetConstant(d, "XD3_COMPLEVEL_9",SWIG_From_int((int)(XD3_COMPLEVEL_9))); -} - |